From 06fa6aeec859e7d15a8d919b80315c9001be0906 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 28 Jul 2020 12:05:53 +0800 Subject: [PATCH 01/59] =?UTF-8?q?=E8=AE=BE=E7=BD=AEclient=E5=88=A0?= =?UTF-8?q?=E9=99=A4key=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sample/pom.xml | 8 +- .../java/com/jd/platform/sample/Cache.java | 136 ++++-------------- .../java/com/jd/platform/sample/Starter.java | 14 ++ .../jd/platform/sample/config/EtcdConfig.java | 26 ++++ .../jd/platform/sample/config/LocalCache.java | 11 -- .../platform/sample/config/RedisConfig.java | 17 +-- .../sample/controller/TestController.java | 84 ++++++----- sample/src/main/resources/application.yml | 12 +- .../netty/server/NodesServerHandler.java | 9 ++ .../hotkey/worker/starters/EtcdStarter.java | 10 +- 10 files changed, 143 insertions(+), 184 deletions(-) create mode 100644 sample/src/main/java/com/jd/platform/sample/config/EtcdConfig.java delete mode 100644 sample/src/main/java/com/jd/platform/sample/config/LocalCache.java diff --git a/sample/pom.xml b/sample/pom.xml index ecd6208..128c969 100644 --- a/sample/pom.xml +++ b/sample/pom.xml @@ -35,10 +35,10 @@ spring-boot-starter-web - - org.springframework.boot - spring-boot-starter-data-redis - + + + + org.springframework.boot diff --git a/sample/src/main/java/com/jd/platform/sample/Cache.java b/sample/src/main/java/com/jd/platform/sample/Cache.java index cfe48e2..9ae2c29 100644 --- a/sample/src/main/java/com/jd/platform/sample/Cache.java +++ b/sample/src/main/java/com/jd/platform/sample/Cache.java @@ -1,119 +1,41 @@ package com.jd.platform.sample; -import com.jd.platform.hotkey.client.callback.JdHotKeyStore; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; -import javax.annotation.Resource; -import java.util.concurrent.CompletableFuture; - /** * @author wuweifeng wrote on 2020-02-21 * @version 1.0 */ @Component public class Cache { - @Resource - private RedisTemplate redisTemplate; - - - public String getFromRedis(String key) { - return redisTemplate.opsForValue().get(key); - } - - //最佳实践： - // - //1 判断用户是否是刷子 - // - // if (JdHotKeyStore.isHotKey(“pin__” + thePin)) { - // //限流他，do your job - // } - //2 判断商品id是否是热点 - // - // - // - // Object skuInfo = JdHotKeyStore.getValue("skuId__" + skuId); - // if(skuInfo == null) { - // - // JdHotKeyStore.smartSet("skuId__" + skuId, theSkuInfo); - // } else { - // - // //使用缓存好的value即可 - // - // } - // - // 或者这样： - // - // - // - // if (JdHotKeyStore.isHotKey(key)) { - // //注意是get，不是getValue。getValue会获取并上报，get是纯粹的本地获取 - // - // Object skuInfo = JdHotKeyStore.get("skuId__" + skuId); - // if(skuInfo == null) { - // - // JdHotKeyStore.smartSet("skuId__" + skuId, theSkuInfo); - // } else { - // - // //使用缓存好的value即可 - // - // } - // - // } - - public String get(String key) { - Object object = JdHotKeyStore.getValue(key); - //如果已经缓存过了 - if (object != null) { - System.out.println("is hot key"); - return object.toString(); - } else { - String value = getFromRedis(key); - JdHotKeyStore.smartSet(key, value); - return value; - } - } - - public void set(String key, String value) { - redisTemplate.opsForValue().set(key, value); - } - - public void remove(String key) { - JdHotKeyStore.remove(key); - //do your job - } - - - private Logger logger = LoggerFactory.getLogger(getClass()); - - -// @PostConstruct - public void test() { - - CompletableFuture.runAsync(() -> { - int i = 0; - while (true) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - logger.info("beat"); -// Object object = JdHotKeyStore.getValue("a"); -// if (object != null) { -// System.err.println("is hot key " + object); -// } else { -// System.err.println("set value"); -// JdHotKeyStore.smartSet("a", "a"); -// } - if (JdHotKeyStore.isHotKey("a")) { - logger.error("isHot"); - } - } - }); - - } +// @Resource +// private RedisTemplate redisTemplate; +// +// +// public String getFromRedis(String key) { +// return redisTemplate.opsForValue().get(key); +// } + + +// public String get(String key) { +// Object object = JdHotKeyStore.getValue(key); +// //如果已经缓存过了 +// if (object != null) { +// System.out.println("is hot key"); +// return object.toString(); +// } else { +// String value = getFromRedis(key); +// JdHotKeyStore.smartSet(key, value); +// return value; +// } +// } +// +// public void set(String key, String value) { +// redisTemplate.opsForValue().set(key, value); +// } +// +// public void remove(String key) { +// JdHotKeyStore.remove(key); +// } } diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index 15a4233..d63e3a0 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -1,10 +1,14 @@ package com.jd.platform.sample; +import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.client.ClientStarter; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.util.List; /** * @author wuweifeng wrote on 2020-01-14 @@ -17,11 +21,21 @@ public class Starter { @Value("${spring.application.name}") private String appName; + @Resource + private IConfigCenter iConfigCenter; + + + @PostConstruct public void init() { ClientStarter.Builder builder = new ClientStarter.Builder(); ClientStarter starter = builder.setAppName(appName).setEtcdServer(etcd).build(); starter.startPipeline(); + + List list = iConfigCenter.getPrefix("/jd/workers/sample/host"); + for (KeyValue keyValue : list) { + System.out.println(keyValue.getKey() + keyValue.getValue().toStringUtf8()); + } } } diff --git a/sample/src/main/java/com/jd/platform/sample/config/EtcdConfig.java b/sample/src/main/java/com/jd/platform/sample/config/EtcdConfig.java new file mode 100644 index 0000000..91feffe --- /dev/null +++ b/sample/src/main/java/com/jd/platform/sample/config/EtcdConfig.java @@ -0,0 +1,26 @@ +package com.jd.platform.sample.config; + +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.configcenter.etcd.JdEtcdBuilder; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-07-27 + */ +@Configuration +public class EtcdConfig { + + @Value("${etcd.server}") + private String etcd; + + + @Bean + public IConfigCenter client() { + //连接多个时，逗号分隔 + return JdEtcdBuilder.build(etcd); + } +} diff --git a/sample/src/main/java/com/jd/platform/sample/config/LocalCache.java b/sample/src/main/java/com/jd/platform/sample/config/LocalCache.java deleted file mode 100644 index 043a9ea..0000000 --- a/sample/src/main/java/com/jd/platform/sample/config/LocalCache.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.jd.platform.sample.config; - -/** - * @author wuweifeng wrote on 2020-02-21 - * @version 1.0 - */ -public class LocalCache { - - - -} diff --git a/sample/src/main/java/com/jd/platform/sample/config/RedisConfig.java b/sample/src/main/java/com/jd/platform/sample/config/RedisConfig.java index b79a8ca..e318e00 100644 --- a/sample/src/main/java/com/jd/platform/sample/config/RedisConfig.java +++ b/sample/src/main/java/com/jd/platform/sample/config/RedisConfig.java @@ -1,21 +1,18 @@ package com.jd.platform.sample.config; -import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.core.StringRedisTemplate; /** * @author wuweifeng wrote on 2017/10/27. */ @Configuration public class RedisConfig { - - @Bean(name = {"redisTemplate", "stringRedisTemplate"}) - public StringRedisTemplate stringRedisTemplate(RedisConnectionFactory factory) { - StringRedisTemplate redisTemplate = new StringRedisTemplate(); - redisTemplate.setConnectionFactory(factory); - return redisTemplate; - } +// +// @Bean(name = {"redisTemplate", "stringRedisTemplate"}) +// public StringRedisTemplate stringRedisTemplate(RedisConnectionFactory factory) { +// StringRedisTemplate redisTemplate = new StringRedisTemplate(); +// redisTemplate.setConnectionFactory(factory); +// return redisTemplate; +// } } diff --git a/sample/src/main/java/com/jd/platform/sample/controller/TestController.java b/sample/src/main/java/com/jd/platform/sample/controller/TestController.java index 230dca0..bf643fb 100644 --- a/sample/src/main/java/com/jd/platform/sample/controller/TestController.java +++ b/sample/src/main/java/com/jd/platform/sample/controller/TestController.java @@ -9,8 +9,6 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.List; /** * @author wuweifeng wrote on 2020-02-21 @@ -28,58 +26,58 @@ public class TestController { /** * 往redis里添加20个key */ - @RequestMapping("addKey") - public Object add(Integer count) { - if (count == null) { - count = 20; - } - for (int i = 0; i < count; i++) { - cache.set("key" + i, "我是一个用来做测试的value：" + i); - } - return "success"; - } +// @RequestMapping("addKey") +// public Object add(Integer count) { +// if (count == null) { +// count = 20; +// } +// for (int i = 0; i < count; i++) { +// cache.set("key" + i, "我是一个用来做测试的value：" + i); +// } +// return "success"; +// } /** * 从redis查询key */ - @RequestMapping("find") - public Object findNormal(Integer count) { - if (count == null) { - count = 20; - } - List values = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - values.add(cache.getFromRedis("key" + i)); - } - return values; - } +// @RequestMapping("find") +// public Object findNormal(Integer count) { +// if (count == null) { +// count = 20; +// } +// List values = new ArrayList<>(count); +// for (int i = 0; i < count; i++) { +// values.add(cache.getFromRedis("key" + i)); +// } +// return values; +// } /** * 使用热key查询，从redis查询key */ - @RequestMapping("findHot") - public Object findWithHotKey(Integer count) { - if (count == null) { - count = 20; - } - List values = new ArrayList<>(count); - for (int i = 0; i < count; i++) { - values.add(cache.get("key" + i)); - } - return values; - } - - - @RequestMapping("hot") - public Object hot(Integer count) { - cache.get("key" + count); - - return 1; - } +// @RequestMapping("findHot") +// public Object findWithHotKey(Integer count) { +// if (count == null) { +// count = 20; +// } +// List values = new ArrayList<>(count); +// for (int i = 0; i < count; i++) { +// values.add(cache.get("key" + i)); +// } +// return values; +// } +// +// +// @RequestMapping("hot") +// public Object hot(Integer count) { +// cache.get("key" + count); +// +// return 1; +// } @RequestMapping("") public Object a() { - if (JdHotKeyStore.isHotKey("a")) { + if (JdHotKeyStore.isHotKey("pin_tianyalei")) { logger.error("isHot"); } else { logger.error("noHot"); diff --git a/sample/src/main/resources/application.yml b/sample/src/main/resources/application.yml index 879608b..41fc686 100644 --- a/sample/src/main/resources/application.yml +++ b/sample/src/main/resources/application.yml @@ -1,13 +1,15 @@ #etcd的地址，如有多个用逗号分隔 etcd: - server: ${etcdServer:https://127.0.0.1:2379} +# server: ${etcdServer:https://127.0.0.1:2379} +# server: http://10.170.161.91:2379 + server: http://open-etcd.jd.com:2000 spring: application: name: sample ###############################---redis---############################## - redis: - host: ${REDIS_HOST:127.0.0.1} - port: ${REDIS_PORT:6379} - password: ${REDIS_PASSWORD:} +# redis: +# host: ${REDIS_HOST:127.0.0.1} +# port: ${REDIS_PORT:6379} +# password: ${REDIS_PASSWORD:} server: port: 9999 \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java index 36cbfaf..64f2a2b 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java @@ -7,6 +7,8 @@ import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; import io.netty.channel.ChannelHandlerContext; import io.netty.channel.SimpleChannelInboundHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; import org.springframework.util.StringUtils; @@ -28,6 +30,8 @@ public class NodesServerHandler extends SimpleChannelInboundHandler { */ private List messageFilters = new ArrayList<>(); + private Logger logger = LoggerFactory.getLogger(getClass()); + @Override protected void channelRead0(ChannelHandlerContext ctx, String message) { if (StringUtils.isEmpty(message)) { @@ -42,6 +46,11 @@ public class NodesServerHandler extends SimpleChannelInboundHandler { } } + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.error("some thing is error , " + cause.getMessage()); + } + @Override public void channelActive(ChannelHandlerContext ctx) throws Exception { super.channelActive(ctx); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java index 895d6a2..9ce9cc9 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java @@ -309,15 +309,17 @@ public class EtcdStarter { } private String buildKey() { - if (StrUtil.isNotEmpty(localAddress)) { - return localAddress; - } String hostName = IpUtils.getHostName(); return ConfigConstant.workersPath + workerPath + "/" + hostName; } private String buildValue() { - String ip = IpUtils.getIp(); + String ip; + if (StrUtil.isNotEmpty(localAddress)) { + ip = localAddress; + } else { + ip = IpUtils.getIp(); + } return ip + MAO + port; } -- Gitee From 2ec9f510b516a014b46ab0263726928dc382da5a Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 29 Jul 2020 11:53:20 +0800 Subject: [PATCH 02/59] =?UTF-8?q?=E5=8A=A0=E5=85=A5ProtoStudff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- worker/pom.xml | 12 +++ .../hotkey/worker/tool/ProtostuffUtils.java | 77 +++++++++++++++++++ worker/src/test/java/Test.java | 38 +++++++++ 3 files changed, 127 insertions(+) create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java create mode 100644 worker/src/test/java/Test.java diff --git a/worker/pom.xml b/worker/pom.xml index 0d64df6..5f234d9 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -19,6 +19,18 @@ + + io.protostuff + protostuff-core + 1.7.2 + + + + io.protostuff + protostuff-runtime + 1.7.2 + + com.jd.platform.hotkey common diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java new file mode 100644 index 0000000..0cb88c5 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java @@ -0,0 +1,77 @@ +package com.jd.platform.hotkey.worker.tool; + +import io.protostuff.LinkedBuffer; +import io.protostuff.ProtostuffIOUtil; +import io.protostuff.Schema; +import io.protostuff.runtime.RuntimeSchema; + +import java.util.Map; +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; + +/** + * + * @author 周志刚 + * @date 2019/6/18 + **/ +public class ProtostuffUtils { + /** + * 避免每次序列化都重新申请Buffer空间 + */ + private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); + /** + * 缓存Schema + */ + private static Map, Schema> schemaCache = new ConcurrentHashMap<>(); + + /** + * 序列化方法，把指定对象序列化成字节数组 + * + * @param obj + * @param + * @return + */ + @SuppressWarnings("unchecked") + public static byte[] serialize(T obj) { + Class clazz = (Class) obj.getClass(); + Schema schema = getSchema(clazz); + byte[] data; + try { + data = ProtostuffIOUtil.toByteArray(obj, schema, buffer); + } finally { + buffer.clear(); + } + + return data; + } + + /** + * 反序列化方法，将字节数组反序列化成指定Class类型 + * + * @param data + * @param clazz + * @param + * @return + */ + public static T deserialize(byte[] data, Class clazz) { + Schema schema = getSchema(clazz); + T obj = schema.newMessage(); + ProtostuffIOUtil.mergeFrom(data, obj, schema); + return obj; + } + + @SuppressWarnings("unchecked") + private static Schema getSchema(Class clazz) { + Schema schema = (Schema) schemaCache.get(clazz); + if (Objects.isNull(schema)) { + //这个schema通过RuntimeSchema进行懒创建并缓存 + //所以可以一直调用RuntimeSchema.getSchema(),这个方法是线程安全的 + schema = RuntimeSchema.getSchema(clazz); + if (Objects.nonNull(schema)) { + schemaCache.put(clazz, schema); + } + } + + return schema; + } +} \ No newline at end of file diff --git a/worker/src/test/java/Test.java b/worker/src/test/java/Test.java new file mode 100644 index 0000000..92a993c --- /dev/null +++ b/worker/src/test/java/Test.java @@ -0,0 +1,38 @@ +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.worker.tool.ProtostuffUtils; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-07-28 + */ +public class Test { + public static void main(String[] args) { + HotKeyMsg hotKeyMsg = new HotKeyMsg(); + hotKeyMsg.setAppName("cartsoa"); + HotKeyModel hotKeyModel = new HotKeyModel(); + hotKeyModel.setCount(1); + hotKeyModel.setKey("pin_xx"); + hotKeyModel.setAppName("cartsoa"); + hotKeyMsg.setBody(FastJsonUtils.convertObjectToJSON(hotKeyModel)); + + byte[] serialize = ProtostuffUtils.serialize(hotKeyMsg); + String msg = FastJsonUtils.convertObjectToJSON(hotKeyMsg); + + long time1 = System.currentTimeMillis(); + for (int i = 0; i < 300000; i++) { + HotKeyMsg hhh = ProtostuffUtils.deserialize(serialize, HotKeyMsg.class); + } + System.out.println(System.currentTimeMillis() - time1); + + long time = System.currentTimeMillis(); + for (int i = 0; i < 300000; i++) { + HotKeyMsg hhh = FastJsonUtils.toBean(msg, HotKeyMsg.class); + } + System.out.println(System.currentTimeMillis() - time); + + + } +} -- Gitee From 41849c8c9181a3ebcfc41a16b2a7581443956b4f Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 31 Jul 2020 14:05:41 +0800 Subject: [PATCH 03/59] =?UTF-8?q?=E5=8A=A0=E5=85=A5ProtoStudff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- client/pom.xml | 4 +- .../client/core/key/NettyKeyPusher.java | 12 +-- .../client/core/worker/WorkerInfoHolder.java | 5 +- .../hotkey/client/netty/NettyClient.java | 13 +-- .../client/netty/NettyClientHandler.java | 18 ++--- common/pom.xml | 15 +++- .../hotkey/common/coder/MsgDecoder.java | 48 +++++++++++ .../hotkey/common/coder/MsgEncoder.java | 30 +++++++ .../hotkey/common/model/HotKeyMsg.java | 33 +++++++- .../hotkey/common/model/MsgBuilder.java | 23 +++--- .../platform/hotkey/common/tool/Constant.java | 2 +- .../hotkey/common}/tool/ProtostuffUtils.java | 14 ++-- pom.xml | 2 +- sample/pom.xml | 2 +- .../java/com/jd/platform/sample/Starter.java | 37 ++++++++- .../sample/controller/TestController.java | 2 +- .../java/com/jd/platform/sample/logback.xml | 60 -------------- sample/src/main/resources/application.yml | 12 ++- sample/src/main/resources/logback-spring.xml | 81 +++++++++++++++++++ worker/pom.xml | 13 +-- .../worker/netty/filter/AppNameFilter.java | 4 +- .../worker/netty/filter/HeartBeatFilter.java | 8 +- .../worker/netty/filter/HotKeyFilter.java | 20 ++--- .../worker/netty/filter/KeyCounterFilter.java | 13 +-- .../hotkey/worker/netty/flush/FlushUtil.java | 23 +++--- .../worker/netty/pusher/AppServerPusher.java | 16 +--- .../worker/netty/server/NodesServer.java | 14 ++-- .../netty/server/NodesServerHandler.java | 13 ++- .../worker/starters/NodesServerStarter.java | 4 - worker/src/test/java/Test.java | 21 ++++- 30 files changed, 360 insertions(+), 202 deletions(-) create mode 100644 common/src/main/java/com/jd/platform/hotkey/common/coder/MsgDecoder.java create mode 100644 common/src/main/java/com/jd/platform/hotkey/common/coder/MsgEncoder.java rename {worker/src/main/java/com/jd/platform/hotkey/worker => common/src/main/java/com/jd/platform/hotkey/common}/tool/ProtostuffUtils.java (72%) delete mode 100644 sample/src/main/java/com/jd/platform/sample/logback.xml create mode 100644 sample/src/main/resources/logback-spring.xml diff --git a/client/pom.xml b/client/pom.xml index 18ef75f..12f5bc4 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -5,12 +5,12 @@ hotkey com.jd.platform.hotkey - 0.0.2-SNAPSHOT + 0.0.4-SNAPSHOT 4.0.0 jar - 0.0.2-SNAPSHOT + 0.0.4-SNAPSHOT hotkey-client diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java index d5afa72..f89def5 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java @@ -6,9 +6,7 @@ import com.jd.platform.hotkey.client.log.JdLogger; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.KeyCountModel; -import com.jd.platform.hotkey.common.model.MsgBuilder; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; import io.netty.channel.Channel; import java.net.InetSocketAddress; @@ -44,8 +42,11 @@ public class NettyKeyPusher implements IKeyPusher { for (Channel channel : map.keySet()) { try { List batch = map.get(channel); - channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_NEW_KEY, FastJsonUtils.convertObjectToJSON(batch)))); + HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.REQUEST_NEW_KEY, Context.APP_NAME); + hotKeyMsg.setHotKeyModels(batch); + channel.writeAndFlush(hotKeyMsg).sync(); } catch (Exception e) { + e.printStackTrace(); try { InetSocketAddress insocket = (InetSocketAddress) channel.remoteAddress(); JdLogger.error(getClass(),"flush error " + insocket.getAddress().getHostAddress()); @@ -77,8 +78,9 @@ public class NettyKeyPusher implements IKeyPusher { for (Channel channel : map.keySet()) { try { List batch = map.get(channel); - channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(Context.APP_NAME, - MessageType.REQUEST_HIT_COUNT, FastJsonUtils.convertObjectToJSON(batch)))); + HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.REQUEST_HIT_COUNT, Context.APP_NAME); + hotKeyMsg.setKeyCountModels(batch); + channel.writeAndFlush(hotKeyMsg); } catch (Exception e) { try { InetSocketAddress insocket = (InetSocketAddress) channel.remoteAddress(); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/worker/WorkerInfoHolder.java b/client/src/main/java/com/jd/platform/hotkey/client/core/worker/WorkerInfoHolder.java index 72471a9..c12182f 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/worker/WorkerInfoHolder.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/worker/WorkerInfoHolder.java @@ -57,10 +57,11 @@ public class WorkerInfoHolder { } public static Channel chooseChannel(String key) { - if (StrUtil.isEmpty(key) || WORKER_HOLDER.size() == 0) { + int size = WORKER_HOLDER.size(); + if (StrUtil.isEmpty(key) || size == 0) { return null; } - int index = Math.abs(key.hashCode() % WORKER_HOLDER.size()); + int index = Math.abs(key.hashCode() % size); return WORKER_HOLDER.get(index).channel; } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java index d2a36a4..6caa7b2 100755 --- a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClient.java @@ -2,8 +2,8 @@ package com.jd.platform.hotkey.client.netty; import com.jd.platform.hotkey.client.core.worker.WorkerInfoHolder; import com.jd.platform.hotkey.client.log.JdLogger; -import com.jd.platform.hotkey.common.coder.Codec; -import com.jd.platform.hotkey.common.coder.NettyCodec; +import com.jd.platform.hotkey.common.coder.MsgDecoder; +import com.jd.platform.hotkey.common.coder.MsgEncoder; import com.jd.platform.hotkey.common.tool.Constant; import io.netty.bootstrap.Bootstrap; import io.netty.buffer.ByteBuf; @@ -13,7 +13,6 @@ import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; import io.netty.channel.socket.nio.NioSocketChannel; import io.netty.handler.codec.DelimiterBasedFrameDecoder; -import io.netty.handler.codec.string.StringDecoder; import io.netty.handler.timeout.IdleStateHandler; import java.util.List; @@ -28,9 +27,6 @@ public class NettyClient { private Bootstrap bootstrap; - private Codec codec = new NettyCodec(); - - public static NettyClient getInstance() { return nettyClient; } @@ -56,9 +52,8 @@ public class NettyClient { ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); ch.pipeline() .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) -// .addLast(codec.newEncoder()) -// .addLast(codec.newDecoder()) - .addLast(new StringDecoder()) + .addLast(new MsgDecoder()) + .addLast(new MsgEncoder()) //10秒没消息时，就发心跳包过去 .addLast(new IdleStateHandler(0, 0, 30)) .addLast(nettyClientHandler); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClientHandler.java b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClientHandler.java index aa78dcd..c16ebf2 100755 --- a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClientHandler.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClientHandler.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.client.netty; +import cn.hutool.core.collection.CollectionUtil; import com.jd.platform.hotkey.client.Context; import com.jd.platform.hotkey.client.callback.ReceiveNewKeyEvent; import com.jd.platform.hotkey.client.core.eventbus.EventBusCenter; @@ -7,10 +8,7 @@ import com.jd.platform.hotkey.client.log.JdLogger; import com.jd.platform.hotkey.client.netty.event.ChannelInactiveEvent; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.model.MsgBuilder; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.Constant; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; import io.netty.channel.Channel; import io.netty.channel.ChannelHandler; import io.netty.channel.ChannelHandlerContext; @@ -22,7 +20,7 @@ import io.netty.handler.timeout.IdleStateEvent; * @author wuweifeng wrote on 2019-11-05. */ @ChannelHandler.Sharable -public class NettyClientHandler extends SimpleChannelInboundHandler { +public class NettyClientHandler extends SimpleChannelInboundHandler { @Override public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { @@ -31,7 +29,7 @@ public class NettyClientHandler extends SimpleChannelInboundHandler { if (idleStateEvent.state() == IdleState.ALL_IDLE) { //向服务端发送消息 - ctx.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.PING, Constant.PING))); + ctx.writeAndFlush(new HotKeyMsg(MessageType.PING, Context.APP_NAME)); } } @@ -41,7 +39,7 @@ public class NettyClientHandler extends SimpleChannelInboundHandler { @Override public void channelActive(ChannelHandlerContext ctx) { JdLogger.info(getClass(), "channelActive:" + ctx.name()); - ctx.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.APP_NAME, Context.APP_NAME))); + ctx.writeAndFlush(new HotKeyMsg(MessageType.APP_NAME, Context.APP_NAME)); } @Override @@ -57,15 +55,17 @@ public class NettyClientHandler extends SimpleChannelInboundHandler { } @Override - protected void channelRead0(ChannelHandlerContext channelHandlerContext, String message) { - HotKeyMsg msg = FastJsonUtils.toBean(message, HotKeyMsg.class); + protected void channelRead0(ChannelHandlerContext channelHandlerContext, HotKeyMsg msg) { if (MessageType.PONG == msg.getMessageType()) { JdLogger.info(getClass(), "heart beat"); return; } if (MessageType.RESPONSE_NEW_KEY == msg.getMessageType()) { JdLogger.info(getClass(), "receive new key : " + msg); - HotKeyModel model = FastJsonUtils.toBean(msg.getBody(), HotKeyModel.class); + if (CollectionUtil.isEmpty(msg.getHotKeyModels())) { + return; + } + HotKeyModel model = msg.getHotKeyModels().get(0); EventBusCenter.getInstance().post(new ReceiveNewKeyEvent(model)); } diff --git a/common/pom.xml b/common/pom.xml index 59a7d76..dec9c4e 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -5,7 +5,7 @@ hotkey com.jd.platform.hotkey - 0.0.2-SNAPSHOT + 0.0.4-SNAPSHOT 4.0.0 @@ -21,9 +21,22 @@ 0.0.14 2.8.0 5.1.0 + 1.7.2 + + io.protostuff + protostuff-core + ${protostuff.version} + + + + io.protostuff + protostuff-runtime + ${protostuff.version} + + cn.hutool hutool-all diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgDecoder.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgDecoder.java new file mode 100644 index 0000000..0710ca1 --- /dev/null +++ b/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgDecoder.java @@ -0,0 +1,48 @@ +package com.jd.platform.hotkey.common.coder; + +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.tool.ProtostuffUtils; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.ByteToMessageDecoder; + +import java.util.List; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-07-29 + */ +public class MsgDecoder extends ByteToMessageDecoder { + @Override + protected void decode(ChannelHandlerContext channelHandlerContext, ByteBuf in, List list) { + try { + + byte[] body = new byte[in.readableBytes()]; //传输正常 + in.readBytes(body); + + list.add(ProtostuffUtils.deserialize(body, HotKeyMsg.class)); + +// if (in.readableBytes() < 4) { +// return; +// } +// in.markReaderIndex(); +// int dataLength = in.readInt(); +// if (dataLength < 0) { +// channelHandlerContext.close(); +// } +// if (in.readableBytes() < dataLength) { +// in.resetReaderIndex(); +// return; +// } +// +// byte[] data = new byte[dataLength]; +// in.readBytes(data); +// +// Object obj = ProtostuffUtils.deserialize(data, HotKeyMsg.class); +// list.add(obj); + } catch (Exception e) { + e.printStackTrace(); + } + } +} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgEncoder.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgEncoder.java new file mode 100644 index 0000000..6cdeca3 --- /dev/null +++ b/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgEncoder.java @@ -0,0 +1,30 @@ +package com.jd.platform.hotkey.common.coder; + +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.tool.Constant; +import com.jd.platform.hotkey.common.tool.ProtostuffUtils; +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import io.netty.handler.codec.MessageToByteEncoder; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-07-30 + */ +public class MsgEncoder extends MessageToByteEncoder { + + @Override + public void encode(ChannelHandlerContext ctx, Object in, ByteBuf out) { + if (in instanceof HotKeyMsg) { + byte[] bytes = ProtostuffUtils.serialize(in); + byte[] delimiter = Constant.DELIMITER.getBytes(); + + byte[] total = new byte[bytes.length + delimiter.length]; + System.arraycopy(bytes, 0, total, 0, bytes.length); + System.arraycopy(delimiter, 0, total, bytes.length, delimiter.length); + + out.writeBytes(total); + } + } +} \ No newline at end of file diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java index a1caf1a..f8ee4e0 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java @@ -2,8 +2,11 @@ package com.jd.platform.hotkey.common.model; import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import java.util.List; + /** * netty通信消息 + * * @author wuweifeng wrote on 2020-01-06 * @version 1.0 */ @@ -16,14 +19,17 @@ public class HotKeyMsg { private String body; - public HotKeyMsg(MessageType messageType, String body) { - this(null, messageType, body); + private List hotKeyModels; + + private List keyCountModels; + + public HotKeyMsg(MessageType messageType) { + this(messageType, null); } - public HotKeyMsg(String appName, MessageType messageType, String body) { + public HotKeyMsg(MessageType messageType, String appName) { this.appName = appName; this.messageType = messageType; - this.body = body; } public HotKeyMsg() { @@ -33,11 +39,30 @@ public class HotKeyMsg { public String toString() { return "HotKeyMsg{" + "magicNumber=" + magicNumber + + ", appName='" + appName + '\'' + ", messageType=" + messageType + ", body='" + body + '\'' + + ", hotKeyModels=" + hotKeyModels + + ", keyCountModels=" + keyCountModels + '}'; } + public List getHotKeyModels() { + return hotKeyModels; + } + + public void setHotKeyModels(List hotKeyModels) { + this.hotKeyModels = hotKeyModels; + } + + public List getKeyCountModels() { + return keyCountModels; + } + + public void setKeyCountModels(List keyCountModels) { + this.keyCountModels = keyCountModels; + } + public String getAppName() { return appName; } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/MsgBuilder.java b/common/src/main/java/com/jd/platform/hotkey/common/model/MsgBuilder.java index f4630a4..fc76264 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/MsgBuilder.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/MsgBuilder.java @@ -1,21 +1,22 @@ package com.jd.platform.hotkey.common.model; -import com.jd.platform.hotkey.common.tool.Constant; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.Unpooled; - /** * @author wuweifeng * @version 1.0 * @date 2020-04-22 */ public class MsgBuilder { - public static ByteBuf buildByteBuf(String msg) { - return Unpooled.copiedBuffer((msg + Constant.DELIMITER).getBytes()); - } +// public static ByteBuf buildByteBuf(String msg) { +// return Unpooled.copiedBuffer((msg + Constant.DELIMITER).getBytes()); +// } - public static ByteBuf buildByteBuf(HotKeyMsg hotKeyMsg) { - return Unpooled.copiedBuffer((FastJsonUtils.convertObjectToJSON(hotKeyMsg) + Constant.DELIMITER).getBytes()); - } +// public static ByteBuf buildByteBuf(HotKeyMsg hotKeyMsg) { +// byte[] bytes = ProtostuffUtils.serialize(hotKeyMsg); +// byte[] delimiter = Constant.DELIMITER.getBytes(); +// +// byte[] bt3 = new byte[bytes.length + delimiter.length]; +// System.arraycopy(bytes, 0, bt3, 0, bytes.length); +// System.arraycopy(delimiter, 0, bt3, bytes.length, delimiter.length); +// return Unpooled.copiedBuffer(bt3); +// } } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java index 5c74754..81a40ec 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java @@ -31,6 +31,6 @@ public class Constant { public static String DEFAULT_DELETE_VALUE = "#[DELETE]#"; //单次包最大2M - public static int MAX_LENGTH = 2 * 1024 * 1024; + public static int MAX_LENGTH = 4 * 1024 * 1024; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/ProtostuffUtils.java similarity index 72% rename from worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java rename to common/src/main/java/com/jd/platform/hotkey/common/tool/ProtostuffUtils.java index 0cb88c5..9425efd 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/ProtostuffUtils.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/ProtostuffUtils.java @@ -1,7 +1,7 @@ -package com.jd.platform.hotkey.worker.tool; +package com.jd.platform.hotkey.common.tool; import io.protostuff.LinkedBuffer; -import io.protostuff.ProtostuffIOUtil; +import io.protostuff.ProtobufIOUtil; import io.protostuff.Schema; import io.protostuff.runtime.RuntimeSchema; @@ -17,8 +17,9 @@ import java.util.concurrent.ConcurrentHashMap; public class ProtostuffUtils { /** * 避免每次序列化都重新申请Buffer空间 + * 这句话在实际生产上没有意义，耗时减少的极小，但高并发下，如果还用这个buffer，会报异常说buffer还没清空，就又被使用了 */ - private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); +// private static LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); /** * 缓存Schema */ @@ -35,9 +36,11 @@ public class ProtostuffUtils { public static byte[] serialize(T obj) { Class clazz = (Class) obj.getClass(); Schema schema = getSchema(clazz); + LinkedBuffer buffer = LinkedBuffer.allocate(LinkedBuffer.DEFAULT_BUFFER_SIZE); byte[] data; try { - data = ProtostuffIOUtil.toByteArray(obj, schema, buffer); + data = ProtobufIOUtil.toByteArray(obj, schema, buffer); +// data = ProtostuffIOUtil.toByteArray(obj, schema, buffer); } finally { buffer.clear(); } @@ -56,7 +59,8 @@ public class ProtostuffUtils { public static T deserialize(byte[] data, Class clazz) { Schema schema = getSchema(clazz); T obj = schema.newMessage(); - ProtostuffIOUtil.mergeFrom(data, obj, schema); + ProtobufIOUtil.mergeFrom(data, obj, schema); +// ProtostuffIOUtil.mergeFrom(data, obj, schema); return obj; } diff --git a/pom.xml b/pom.xml index 2522063..c89611d 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ pom com.jd.platform.hotkey hotkey - 0.0.2-SNAPSHOT + 0.0.4-SNAPSHOT hotkey Jd HotKey diff --git a/sample/pom.xml b/sample/pom.xml index 128c969..687331e 100644 --- a/sample/pom.xml +++ b/sample/pom.xml @@ -10,7 +10,7 @@ com.jd.platform sample - 0.0.2-SNAPSHOT + 0.0.4-SNAPSHOT sample Demo project for Spring Boot diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index d63e3a0..0356ce2 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -3,6 +3,8 @@ package com.jd.platform.sample; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.client.ClientStarter; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -21,21 +23,52 @@ public class Starter { @Value("${spring.application.name}") private String appName; + @Value("${total.count}") + private int count; + @Value("${total.sleep}") + private int sleep; + @Value("${total.period}") + private long period; + @Resource private IConfigCenter iConfigCenter; - + private Logger logger = LoggerFactory.getLogger(getClass()); @PostConstruct public void init() { ClientStarter.Builder builder = new ClientStarter.Builder(); - ClientStarter starter = builder.setAppName(appName).setEtcdServer(etcd).build(); + ClientStarter starter = builder.setAppName(appName).setEtcdServer(etcd).setPushPeriod(period).build(); starter.startPipeline(); List list = iConfigCenter.getPrefix("/jd/workers/sample/host"); for (KeyValue keyValue : list) { System.out.println(keyValue.getKey() + keyValue.getValue().toStringUtf8()); } + + try { + Thread.sleep(3000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + +// new Thread(() -> { +// long i1 = 0; +// while (true) { +// if (JdHotKeyStore.getValue(i1 + "") == null) { +// i1++; +// if (i1 % count == 0) { +// try { +// Thread.sleep(sleep); +// logger.info("number: " + i1); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// } +// } +// } +// +// }).start(); } } diff --git a/sample/src/main/java/com/jd/platform/sample/controller/TestController.java b/sample/src/main/java/com/jd/platform/sample/controller/TestController.java index bf643fb..e72b180 100644 --- a/sample/src/main/java/com/jd/platform/sample/controller/TestController.java +++ b/sample/src/main/java/com/jd/platform/sample/controller/TestController.java @@ -77,7 +77,7 @@ public class TestController { @RequestMapping("") public Object a() { - if (JdHotKeyStore.isHotKey("pin_tianyalei")) { + if (JdHotKeyStore.isHotKey("VENDER_ID_2111")) { logger.error("isHot"); } else { logger.error("noHot"); diff --git a/sample/src/main/java/com/jd/platform/sample/logback.xml b/sample/src/main/java/com/jd/platform/sample/logback.xml deleted file mode 100644 index 918e0a0..0000000 --- a/sample/src/main/java/com/jd/platform/sample/logback.xml +++ /dev/null @@ -1,60 +0,0 @@ - - - - - - - - - - UTF-8 - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n - - - - - - UTF-8 - - ${LOG_HOME}/${appName}.log - - - - ${LOG_HOME}/${appName}-%d{yyyy-MM-dd}-%i.log - - - 30 - - - 512MB - - - - - %d{yyyy-MM-dd HH:mm:ss.SSS} [ %thread ] - [ %-5level ] [%logger{50} : %line ] - %msg%n - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/sample/src/main/resources/application.yml b/sample/src/main/resources/application.yml index 41fc686..a8e2a8c 100644 --- a/sample/src/main/resources/application.yml +++ b/sample/src/main/resources/application.yml @@ -1,11 +1,15 @@ #etcd的地址，如有多个用逗号分隔 etcd: -# server: ${etcdServer:https://127.0.0.1:2379} -# server: http://10.170.161.91:2379 - server: http://open-etcd.jd.com:2000 + server: ${etcdServer:https://127.0.0.1:2379} +# server: ${etcdServer:http://10.170.161.91:2379｝ +# server: http://open-etcd.jd.com:2000 +total: + count: ${count:40000} + sleep: ${sleep:1000} + period: ${period:10} spring: application: - name: sample + name: ${name:sample} ###############################---redis---############################## # redis: # host: ${REDIS_HOST:127.0.0.1} diff --git a/sample/src/main/resources/logback-spring.xml b/sample/src/main/resources/logback-spring.xml new file mode 100644 index 0000000..3eee221 --- /dev/null +++ b/sample/src/main/resources/logback-spring.xml @@ -0,0 +1,81 @@ + + + + + + + + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n + + + + + + + ${LOG_HOME}/${appName}.log + + + ${LOG_HOME}/${appName}-%d{yyyy-MM-dd}-%i.log + + 365 + + + 100MB + + + + + %d{yyyy-MM-dd HH:mm:ss.SSS} [ %thread ] - [ %-5level ] [ %logger{50} : %line ] - %msg%n + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/worker/pom.xml b/worker/pom.xml index 5f234d9..45897a4 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -10,7 +10,7 @@ worker - 0.0.2-SNAPSHOT + 0.0.4-SNAPSHOT worker jar @@ -19,17 +19,6 @@ - - io.protostuff - protostuff-core - 1.7.2 - - - - io.protostuff - protostuff-runtime - 1.7.2 - com.jd.platform.hotkey diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java index 4d81f25..f3b761d 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java @@ -2,8 +2,8 @@ package com.jd.platform.hotkey.worker.netty.filter; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.common.tool.NettyIpUtil; +import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import io.netty.channel.ChannelHandlerContext; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; @@ -25,7 +25,7 @@ public class AppNameFilter implements INettyMsgFilter { @Override public boolean chain(HotKeyMsg message, ChannelHandlerContext ctx) { if (MessageType.APP_NAME == message.getMessageType()) { - String appName = message.getBody(); + String appName = message.getAppName(); if (clientEventListener != null) { clientEventListener.newClient(appName, NettyIpUtil.clientIp(ctx), ctx); } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java index a013d53..82112ca 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java @@ -1,16 +1,12 @@ package com.jd.platform.hotkey.worker.netty.filter; import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.model.MsgBuilder; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.worker.netty.flush.FlushUtil; import io.netty.channel.ChannelHandlerContext; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; -import static com.jd.platform.hotkey.common.tool.Constant.PONG; - /** * 心跳包处理 * @author wuweifeng wrote on 2019-12-11 @@ -19,11 +15,11 @@ import static com.jd.platform.hotkey.common.tool.Constant.PONG; @Component @Order(1) public class HeartBeatFilter implements INettyMsgFilter { + @Override public boolean chain(HotKeyMsg message, ChannelHandlerContext ctx) { if (MessageType.PING == message.getMessageType()) { - String hotMsg = FastJsonUtils.convertObjectToJSON(new HotKeyMsg(MessageType.PONG, PONG)); - FlushUtil.flush(ctx, MsgBuilder.buildByteBuf(hotMsg)); + FlushUtil.flush(ctx, new HotKeyMsg(MessageType.PONG)); return false; } return true; diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index 080a9d9..2e37f08 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -1,10 +1,10 @@ package com.jd.platform.hotkey.worker.netty.filter; +import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.date.SystemClock; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.keydispatcher.KeyProducer; import com.jd.platform.hotkey.worker.mq.IMqMessageReceiver; @@ -40,7 +40,7 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { if (MessageType.REQUEST_NEW_KEY == message.getMessageType()) { totalReceiveKeyCount.incrementAndGet(); - publishMsg(message.getBody(), ctx); + publishMsg(message, ctx); return false; } @@ -50,21 +50,15 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { @Override public void receive(String msg) { - publishMsg(msg, null); +// publishMsg(msg, null); } - private void publishMsg(String message, ChannelHandlerContext ctx) { - //这个是给测试用的，实际走的是下面那个 - if (message.startsWith("{")) { - HotKeyModel model = FastJsonUtils.toBean(message, HotKeyModel.class); - if (WhiteListHolder.contains(model.getKey())) { - return; - } - keyProducer.push(model); + private void publishMsg(HotKeyMsg message, ChannelHandlerContext ctx) { + //老版的用的单个HotKeyModel，新版用的数组 + List models = message.getHotKeyModels(); + if (CollectionUtil.isEmpty(models)) { return; } - //老版的用的单个HotKeyModel，新版用的数组 - List models = FastJsonUtils.toList(message, HotKeyModel.class); for (HotKeyModel model : models) { //白名单key不处理 if (WhiteListHolder.contains(model.getKey())) { diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java index d8eae48..0022157 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java @@ -1,11 +1,11 @@ package com.jd.platform.hotkey.worker.netty.filter; +import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.date.SystemClock; import cn.hutool.core.util.StrUtil; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.KeyCountModel; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.counter.KeyCountItem; import com.jd.platform.hotkey.worker.mq.IMqMessageReceiver; @@ -44,7 +44,7 @@ public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { if (StrUtil.isEmpty(message.getAppName())) { message.setAppName(workerPath); } - publishMsg(message.getAppName(), message.getBody(), ctx); + publishMsg(message.getAppName(), message, ctx); return false; } @@ -54,12 +54,15 @@ public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { @Override public void receive(String msg) { - publishMsg("", msg, null); +// publishMsg("", msg, null); } - private void publishMsg(String appName, String message, ChannelHandlerContext ctx) { + private void publishMsg(String appName, HotKeyMsg message, ChannelHandlerContext ctx) { //老版的用的单个HotKeyModel，新版用的数组 - List models = FastJsonUtils.toList(message, KeyCountModel.class); + List models = message.getKeyCountModels(); + if (CollectionUtil.isEmpty(models)) { + return; + } long timeOut = SystemClock.now() - models.get(0).getCreateTime(); //超时5秒以上的就不处理了，因为client是每10秒发送一次，所以最迟15秒以后的就不处理了 if (timeOut > InitConstant.timeOut + 10000) { diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java index cdc1299..c9f3ed0 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java @@ -1,6 +1,6 @@ package com.jd.platform.hotkey.worker.netty.flush; -import io.netty.buffer.ByteBuf; +import com.jd.platform.hotkey.common.model.HotKeyMsg; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -15,20 +15,21 @@ public class FlushUtil { /** * 往channel里输出消息 */ - public static void flush(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf) { - if (channelHandlerContext.channel().isWritable()) { - channelHandlerContext.channel().writeAndFlush(byteBuf).addListener(future -> { - if (!future.isSuccess()) { - logger.warn("flush error " + future.cause().getMessage()); - } - }); - } else { + + public static void flush(ChannelHandlerContext channelHandlerContext, HotKeyMsg hotKeyMsg) { +// if (channelHandlerContext.channel().isWritable()) { +// channelHandlerContext.channel().writeAndFlush(hotKeyMsg).addListener(future -> { +// if (!future.isSuccess()) { +// logger.warn("flush error " + future.cause().getMessage()); +// } +// }); +// } else { try { //同步发送 - channelHandlerContext.channel().writeAndFlush(byteBuf).sync(); + channelHandlerContext.channel().writeAndFlush(hotKeyMsg).sync(); } catch (InterruptedException e) { logger.error("flush error " + e.getMessage()); } - } +// } } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java index db89b85..0743e3e 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java @@ -2,16 +2,14 @@ package com.jd.platform.hotkey.worker.netty.pusher; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.model.MsgBuilder; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.worker.model.AppInfo; import com.jd.platform.hotkey.worker.netty.flush.FlushUtil; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; -import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import org.springframework.stereotype.Component; +import java.util.Arrays; import java.util.Map; /** @@ -31,18 +29,12 @@ public class AppServerPusher implements IPusher { if (model.getAppName().equals(appInfo.getAppName())) { Map map = appInfo.getMap(); - HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.RESPONSE_NEW_KEY, FastJsonUtils.convertObjectToJSON(model)); - String hotMsg = FastJsonUtils.convertObjectToJSON(hotKeyMsg); - -// for (ChannelHandlerContext channel : map.values()) { -// ByteBuf byteBuf = MsgBuilder.buildByteBuf(hotMsg); -// FlushUtil.flush(channel, byteBuf); -// } + HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.RESPONSE_NEW_KEY); + hotKeyMsg.setHotKeyModels(Arrays.asList(model)); //并行发送 map.values().parallelStream().forEach(channel -> { - ByteBuf byteBuf = MsgBuilder.buildByteBuf(hotMsg); - FlushUtil.flush(channel, byteBuf); + FlushUtil.flush(channel, hotKeyMsg); }); return; diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java index 6ea9b4f..606e07f 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.worker.netty.server; -import com.jd.platform.hotkey.common.coder.Codec; +import com.jd.platform.hotkey.common.coder.MsgDecoder; +import com.jd.platform.hotkey.common.coder.MsgEncoder; import com.jd.platform.hotkey.common.tool.Constant; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; @@ -12,7 +13,6 @@ import io.netty.channel.*; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.nio.NioServerSocketChannel; import io.netty.handler.codec.DelimiterBasedFrameDecoder; -import io.netty.handler.codec.string.StringDecoder; import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; @@ -26,7 +26,6 @@ import java.util.List; public class NodesServer { private IClientChangeListener clientChangeListener; private List messageFilters; - private Codec codec; public void startNettyServer(int port) throws Exception { //boss单线程 @@ -47,6 +46,7 @@ public class NodesServer { //等待服务器监听端口关闭 future.channel().closeFuture().sync(); } catch (Exception e) { + e.printStackTrace(); //do nothing System.out.println("netty stop"); } finally { @@ -70,9 +70,8 @@ public class NodesServer { ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); ch.pipeline() .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) -// .addLast(codec.newEncoder()) -// .addLast(codec.newDecoder()) - .addLast(new StringDecoder()) + .addLast(new MsgDecoder()) + .addLast(new MsgEncoder()) .addLast(serverHandler); } } @@ -85,7 +84,4 @@ public class NodesServer { this.messageFilters = messageFilters; } - public void setCodec(Codec codec) { - this.codec = codec; - } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java index 64f2a2b..37b7bb2 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java @@ -1,7 +1,6 @@ package com.jd.platform.hotkey.worker.netty.server; import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; @@ -10,7 +9,6 @@ import io.netty.channel.SimpleChannelInboundHandler; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; import java.util.ArrayList; import java.util.List; @@ -20,7 +18,7 @@ import java.util.List; * * @author wuweifeng wrote on 2019-11-05. */ -public class NodesServerHandler extends SimpleChannelInboundHandler { +public class NodesServerHandler extends SimpleChannelInboundHandler { /** * 客户端状态监听器 */ @@ -33,21 +31,22 @@ public class NodesServerHandler extends SimpleChannelInboundHandler { private Logger logger = LoggerFactory.getLogger(getClass()); @Override - protected void channelRead0(ChannelHandlerContext ctx, String message) { - if (StringUtils.isEmpty(message)) { + protected void channelRead0(ChannelHandlerContext ctx, HotKeyMsg msg) { + if (msg == null) { return; } - HotKeyMsg msg = FastJsonUtils.toBean(message, HotKeyMsg.class); for (INettyMsgFilter messageFilter : messageFilters) { boolean doNext = messageFilter.chain(msg, ctx); if (!doNext) { return; } } + } @Override - public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) { + cause.printStackTrace(); logger.error("some thing is error , " + cause.getMessage()); } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java index 0b9c522..5759f5e 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.worker.starters; -import com.jd.platform.hotkey.common.coder.Codec; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; import com.jd.platform.hotkey.worker.netty.server.NodesServer; @@ -29,8 +28,6 @@ public class NodesServerStarter { private IClientChangeListener iClientChangeListener; @Resource private List messageFilters; - @Resource - private Codec codec; @PostConstruct public void start() { @@ -40,7 +37,6 @@ public class NodesServerStarter { NodesServer nodesServer = new NodesServer(); nodesServer.setClientChangeListener(iClientChangeListener); nodesServer.setMessageFilters(messageFilters); - nodesServer.setCodec(codec); try { nodesServer.startNettyServer(port); } catch (Exception e) { diff --git a/worker/src/test/java/Test.java b/worker/src/test/java/Test.java index 92a993c..207f918 100644 --- a/worker/src/test/java/Test.java +++ b/worker/src/test/java/Test.java @@ -1,7 +1,11 @@ import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.KeyCountModel; import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import com.jd.platform.hotkey.worker.tool.ProtostuffUtils; +import com.jd.platform.hotkey.common.tool.ProtostuffUtils; + +import java.util.ArrayList; +import java.util.List; /** * @author wuweifeng @@ -16,10 +20,20 @@ public class Test { hotKeyModel.setCount(1); hotKeyModel.setKey("pin_xx"); hotKeyModel.setAppName("cartsoa"); - hotKeyMsg.setBody(FastJsonUtils.convertObjectToJSON(hotKeyModel)); + + List hotKeyModels = new ArrayList<>(); + for (int i = 0; i < 500; i++) { + hotKeyModels.add(hotKeyModel); + } + hotKeyMsg.setHotKeyModels(hotKeyModels); + + KeyCountModel keyCountModel = new KeyCountModel(); + keyCountModel.setHotHitCount(11); + List keyCountModels = new ArrayList<>(); + keyCountModels.add(keyCountModel); +// hotKeyMsg.setKeyCountModels(keyCountModels); byte[] serialize = ProtostuffUtils.serialize(hotKeyMsg); - String msg = FastJsonUtils.convertObjectToJSON(hotKeyMsg); long time1 = System.currentTimeMillis(); for (int i = 0; i < 300000; i++) { @@ -27,6 +41,7 @@ public class Test { } System.out.println(System.currentTimeMillis() - time1); + String msg = FastJsonUtils.convertObjectToJSON(hotKeyMsg); long time = System.currentTimeMillis(); for (int i = 0; i < 300000; i++) { HotKeyMsg hhh = FastJsonUtils.toBean(msg, HotKeyMsg.class); -- Gitee From 908426abd4f0a9b18db30990a9a1d95d817948fb Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 31 Jul 2020 14:13:03 +0800 Subject: [PATCH 04/59] =?UTF-8?q?count=E4=B9=9F=E5=90=8C=E6=AD=A5=E5=8F=91?= =?UTF-8?q?=E9=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java index f89def5..988c3cc 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java @@ -80,7 +80,7 @@ public class NettyKeyPusher implements IKeyPusher { List batch = map.get(channel); HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.REQUEST_HIT_COUNT, Context.APP_NAME); hotKeyMsg.setKeyCountModels(batch); - channel.writeAndFlush(hotKeyMsg); + channel.writeAndFlush(hotKeyMsg).sync(); } catch (Exception e) { try { InetSocketAddress insocket = (InetSocketAddress) channel.remoteAddress(); -- Gitee From ac172ba28dfbce1ecb9e58e142b3c064f610b3b8 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 31 Jul 2020 16:53:48 +0800 Subject: [PATCH 05/59] =?UTF-8?q?=E5=8A=A0=E5=85=A5ProtoStudff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jd/platform/hotkey/common/tool/ProtostuffUtils.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/ProtostuffUtils.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/ProtostuffUtils.java index 9425efd..c7f3928 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/ProtostuffUtils.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/ProtostuffUtils.java @@ -11,8 +11,8 @@ import java.util.concurrent.ConcurrentHashMap; /** * - * @author 周志刚 - * @date 2019/6/18 + * @author wuweifeng10 + * @date 2020/7/18 **/ public class ProtostuffUtils { /** @@ -50,11 +50,6 @@ public class ProtostuffUtils { /** * 反序列化方法，将字节数组反序列化成指定Class类型 - * - * @param data - * @param clazz - * @param - * @return */ public static T deserialize(byte[] data, Class clazz) { Schema schema = getSchema(clazz); -- Gitee From eb9aa5012fe915f53b05e44968b66b368fed05d3 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Sat, 1 Aug 2020 16:56:16 +0800 Subject: [PATCH 06/59] =?UTF-8?q?=E8=A7=84=E5=88=99=E6=8B=89=E5=AE=8C?= =?UTF-8?q?=E5=90=8E=EF=BC=8C=E5=86=8D=E6=8B=89=E5=B7=B2=E5=AD=98=E5=9C=A8?= =?UTF-8?q?=E7=9A=84key?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java index c129fb1..4c0b1b4 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java @@ -38,8 +38,6 @@ public class EtcdStarter { fetchRule(); - fetchExistHotKey(); - // startWatchWorker(); startWatchRule(); @@ -243,6 +241,9 @@ public class EtcdStarter { JdLogger.info(getClass(), "trying to connect to etcd and fetch rule info"); boolean success = fetchRuleFromEtcd(); if (success) { + //拉取已经存在的热key + fetchExistHotKey(); + scheduledExecutorService.shutdown(); } -- Gitee From 2ee70e75fba7d34d468f2eb0546c5c01e861ad33 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 6 Aug 2020 18:45:43 +0800 Subject: [PATCH 07/59] =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java | 1 - 1 file changed, 1 deletion(-) diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java index ec8a185..d3801e8 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineCacheHolder.java @@ -42,7 +42,6 @@ public class CaffeineCacheHolder { public static void clearCacheByAppName(String appName) { if(CACHE_MAP.get(appName) != null) { CACHE_MAP.get(appName).invalidateAll(); - CACHE_MAP.put(appName, null); } } -- Gitee From 478a2f53e279265c75d7959e27b6ca31e91e64ee Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 7 Aug 2020 11:53:37 +0800 Subject: [PATCH 08/59] =?UTF-8?q?=E9=87=87=E7=94=A8ChannelGroup=E6=9B=BF?= =?UTF-8?q?=E6=8D=A2=E4=B9=8B=E5=89=8D=E7=9A=84hashmap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- worker/pom.xml | 1 - .../keydispatcher/DispatcherConfig.java | 2 +- .../platform/hotkey/worker/model/AppInfo.java | 42 ++++++++++--------- .../netty/client/ClientChangeListener.java | 22 ++++------ .../netty/client/IClientChangeListener.java | 2 +- .../worker/netty/filter/AppNameFilter.java | 4 +- .../worker/netty/pusher/AppServerPusher.java | 10 +---- .../netty/server/NodesServerHandler.java | 3 +- .../hotkey/worker/starters/EtcdStarter.java | 5 +-- 9 files changed, 38 insertions(+), 53 deletions(-) diff --git a/worker/pom.xml b/worker/pom.xml index 45897a4..72c8aee 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -19,7 +19,6 @@ - com.jd.platform.hotkey common diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java index 3d620b1..defcf67 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java @@ -39,7 +39,7 @@ public class DispatcherConfig { public Consumer consumer() { int nowCount = CpuNum.workerCount(); //将实际值赋给static变量 - if(threadCount != 0) { + if (threadCount != 0) { nowCount = threadCount; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/model/AppInfo.java b/worker/src/main/java/com/jd/platform/hotkey/worker/model/AppInfo.java index 4e106a3..599edde 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/model/AppInfo.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/model/AppInfo.java @@ -1,9 +1,9 @@ package com.jd.platform.hotkey.worker.model; import io.netty.channel.ChannelHandlerContext; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; +import io.netty.channel.group.ChannelGroup; +import io.netty.channel.group.DefaultChannelGroup; +import io.netty.util.concurrent.GlobalEventExecutor; /** * @author wuweifeng wrote on 2019-12-05 @@ -15,31 +15,33 @@ public class AppInfo { */ private String appName; /** - * 客户端ip 和 channel的映射关系 + * 某app的全部channel */ - private Map map = new ConcurrentHashMap<>(); - - @Override - public String toString() { - return "AppInfo{" + - "appName='" + appName + '\'' + - ", map=" + map + - '}'; + private ChannelGroup channelGroup; + + public AppInfo(String appName) { + this.appName = appName; + channelGroup = new DefaultChannelGroup(appName, GlobalEventExecutor.INSTANCE); } - public String getAppName() { - return appName; + public void groupPush(Object object) { + channelGroup.writeAndFlush(object); } - public void setAppName(String appName) { - this.appName = appName; + public void add(ChannelHandlerContext ctx) { + channelGroup.add(ctx.channel()); } - public Map getMap() { - return map; + public void remove(ChannelHandlerContext ctx) { + channelGroup.remove(ctx.channel()); } - public void setMap(Map map) { - this.map = map; + public String getAppName() { + return appName; } + + public int size() { + return channelGroup.size(); + } + } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/ClientChangeListener.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/ClientChangeListener.java index daca0c9..f2362b7 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/ClientChangeListener.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/ClientChangeListener.java @@ -1,13 +1,12 @@ package com.jd.platform.hotkey.worker.netty.client; +import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.model.AppInfo; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Map; - /** * 对客户端的管理，新来、断线的管理 * @@ -20,7 +19,7 @@ public class ClientChangeListener implements IClientChangeListener { private static final String NEW_CLIENT = "监听到事件"; private static final String NEW_CLIENT_JOIN = "new client join"; - private static final String CLIENT_LOSE = "client removed"; + private static final String CLIENT_LOSE = "client removed "; /** * 客户端新增 @@ -33,29 +32,24 @@ public class ClientChangeListener implements IClientChangeListener { for (AppInfo appInfo : ClientInfoHolder.apps) { if (appName.equals(appInfo.getAppName())) { appExist = true; - appInfo.getMap().put(ip, ctx); + appInfo.add(ctx); break; } } if (!appExist) { - AppInfo appInfo = new AppInfo(); - appInfo.setAppName(appName); + AppInfo appInfo = new AppInfo(appName); ClientInfoHolder.apps.add(appInfo); - appInfo.getMap().put(ip, ctx); + appInfo.add(ctx); } logger.info(NEW_CLIENT_JOIN); } @Override - public synchronized void loseClient(String ip) { + public synchronized void loseClient(ChannelHandlerContext ctx) { for (AppInfo appInfo : ClientInfoHolder.apps) { - Map map = appInfo.getMap(); - if (map.containsKey(ip)) { - map.remove(ip); - break; - } + appInfo.remove(ctx); } - logger.info(CLIENT_LOSE); + logger.info(CLIENT_LOSE + NettyIpUtil.clientIp(ctx)); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/IClientChangeListener.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/IClientChangeListener.java index a6656b8..b97825b 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/IClientChangeListener.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/client/IClientChangeListener.java @@ -15,5 +15,5 @@ public interface IClientChangeListener { /** * 客户端掉线 */ - void loseClient(String channelId); + void loseClient(ChannelHandlerContext ctx); } \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java index f3b761d..4d81f25 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java @@ -2,8 +2,8 @@ package com.jd.platform.hotkey.worker.netty.filter; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; +import com.jd.platform.hotkey.common.tool.NettyIpUtil; import io.netty.channel.ChannelHandlerContext; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; @@ -25,7 +25,7 @@ public class AppNameFilter implements INettyMsgFilter { @Override public boolean chain(HotKeyMsg message, ChannelHandlerContext ctx) { if (MessageType.APP_NAME == message.getMessageType()) { - String appName = message.getAppName(); + String appName = message.getBody(); if (clientEventListener != null) { clientEventListener.newClient(appName, NettyIpUtil.clientIp(ctx), ctx); } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java index 0743e3e..5a2e150 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java @@ -4,13 +4,10 @@ import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.worker.model.AppInfo; -import com.jd.platform.hotkey.worker.netty.flush.FlushUtil; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; -import io.netty.channel.ChannelHandlerContext; import org.springframework.stereotype.Component; import java.util.Arrays; -import java.util.Map; /** * 推送到各客户端服务器 @@ -27,15 +24,12 @@ public class AppServerPusher implements IPusher { public void push(HotKeyModel model) { for (AppInfo appInfo : ClientInfoHolder.apps) { if (model.getAppName().equals(appInfo.getAppName())) { - Map map = appInfo.getMap(); HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.RESPONSE_NEW_KEY); hotKeyMsg.setHotKeyModels(Arrays.asList(model)); - //并行发送 - map.values().parallelStream().forEach(channel -> { - FlushUtil.flush(channel, hotKeyMsg); - }); + //整个app全部发送 + appInfo.groupPush(hotKeyMsg); return; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java index 37b7bb2..658b4c0 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java @@ -1,7 +1,6 @@ package com.jd.platform.hotkey.worker.netty.server; import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; import io.netty.channel.ChannelHandlerContext; @@ -58,7 +57,7 @@ public class NodesServerHandler extends SimpleChannelInboundHandler { @Override public void channelInactive(ChannelHandlerContext ctx) throws Exception { if (clientEventListener != null) { - clientEventListener.loseClient(NettyIpUtil.clientIp(ctx)); + clientEventListener.loseClient(ctx); } ctx.close(); super.channelInactive(ctx); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java index 9ce9cc9..9fe62a4 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java @@ -20,7 +20,6 @@ import com.jd.platform.hotkey.worker.rule.KeyRuleHolder; import com.jd.platform.hotkey.worker.tool.AsyncPool; import com.jd.platform.hotkey.worker.tool.InitConstant; import io.grpc.StatusRuntimeException; -import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -31,7 +30,6 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.annotation.Resource; import java.util.List; -import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -196,8 +194,7 @@ public class EtcdStarter { String ip = IpUtils.getIp(); for (AppInfo appInfo : ClientInfoHolder.apps) { String appName = appInfo.getAppName(); - Map map = appInfo.getMap(); - int count = map.values().size(); + int count = appInfo.size(); //即便是full gc也不能超过3秒 configCenter.putAndGrant(ConfigConstant.clientCountPath + appName + "/" + ip, count + "", 13); } -- Gitee From 28b10988969a5b0d9b8251931a6f7caab253abf9 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Tue, 18 Aug 2020 20:55:30 +0800 Subject: [PATCH 09/59] update LICENSE. --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 261eeb9..48d4f57 100644 --- a/LICENSE +++ b/LICENSE @@ -186,7 +186,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright [yyyy] [name of copyright owner] + Copyright [2020] [JD.com, Inc.] Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. -- Gitee From 4551a03d8d470822cb6275405b6c048a8bb1e103 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 21 Aug 2020 18:41:58 +0800 Subject: [PATCH 10/59] appName --- .../jd/platform/hotkey/worker/netty/filter/AppNameFilter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java index 4d81f25..4034619 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/AppNameFilter.java @@ -25,7 +25,7 @@ public class AppNameFilter implements INettyMsgFilter { @Override public boolean chain(HotKeyMsg message, ChannelHandlerContext ctx) { if (MessageType.APP_NAME == message.getMessageType()) { - String appName = message.getBody(); + String appName = message.getAppName(); if (clientEventListener != null) { clientEventListener.newClient(appName, NettyIpUtil.clientIp(ctx), ctx); } -- Gitee From 4a6711dd6d5671daead593b75ee09b56449087f4 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Mon, 24 Aug 2020 16:21:06 +0800 Subject: [PATCH 11/59] =?UTF-8?q?=E4=BF=AE=E6=94=B9worker=E7=9A=84io?= =?UTF-8?q?=E7=BA=BF=E7=A8=8B=E6=95=B0=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/jd/platform/sample/Starter.java | 35 ++++++++++--------- .../keydispatcher/DispatcherConfig.java | 4 +++ .../platform/hotkey/worker/tool/CpuNum.java | 16 ++++----- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/sample/src/main/java/com/jd/platform/sample/Starter.java b/sample/src/main/java/com/jd/platform/sample/Starter.java index 0356ce2..43baaae 100644 --- a/sample/src/main/java/com/jd/platform/sample/Starter.java +++ b/sample/src/main/java/com/jd/platform/sample/Starter.java @@ -2,6 +2,7 @@ package com.jd.platform.sample; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.client.ClientStarter; +import com.jd.platform.hotkey.client.callback.JdHotKeyStore; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -52,23 +53,23 @@ public class Starter { e.printStackTrace(); } -// new Thread(() -> { -// long i1 = 0; -// while (true) { -// if (JdHotKeyStore.getValue(i1 + "") == null) { -// i1++; -// if (i1 % count == 0) { -// try { -// Thread.sleep(sleep); -// logger.info("number: " + i1); -// } catch (InterruptedException e) { -// e.printStackTrace(); -// } -// } -// } -// } -// -// }).start(); + new Thread(() -> { + long i1 = 0; + while (true) { + if (JdHotKeyStore.getValue(i1 + "") == null) { + i1++; + if (i1 % count == 0) { + try { + Thread.sleep(sleep); + logger.info("number: " + i1); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + } + + }).start(); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java index defcf67..7e3094a 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/DispatcherConfig.java @@ -41,6 +41,10 @@ public class DispatcherConfig { //将实际值赋给static变量 if (threadCount != 0) { nowCount = threadCount; + } else { + if (nowCount >= 8) { + nowCount = nowCount / 2; + } } List consumerList = new ArrayList<>(); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java index acaa70e..55bd842 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java @@ -13,17 +13,13 @@ public class CpuNum { public static int workerCount() { //取cpu核数，新版jdk在docker里取的就是真实分配的，老版jdk取的是宿主机的，可能特别大，如32核 int count = Runtime.getRuntime().availableProcessors(); -// if (isNewerVersion()) { -// if (count >= 4) { -// count = count / 2; -// } -// } else { -// if (count >= 8) { -// count = 4; -// } -// } - if (count >= 4) { + if (isNewerVersion()) { + return count; + } else { count = count / 2; + if (count == 0) { + count = 1; + } } return count; } -- Gitee From 70a4efbf6b737ae2158fb03903d7416a966a4f89 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 25 Aug 2020 12:22:13 +0800 Subject: [PATCH 12/59] =?UTF-8?q?=E9=87=87=E7=94=A8=E7=BA=BF=E7=A8=8B?= =?UTF-8?q?=E6=B1=A0=E6=8E=A8=E9=80=81=E5=88=B0etcd?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/pom.xml | 2 +- .../hotkey/worker/netty/pusher/EtcdPusher.java | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index dec9c4e..8b30a03 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -18,7 +18,7 @@ 3.4.2 1.1.7.3 1.2.70 - 0.0.14 + 0.0.16 2.8.0 5.1.0 1.7.2 diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java index 627d3eb..aa1971b 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java @@ -8,9 +8,12 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.util.UUID; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** + * 将热key推送到etcd，dashboard监听后留做备份 * @author wuweifeng wrote on 2020-02-24 * @version 1.0 */ @@ -19,12 +22,20 @@ public class EtcdPusher implements IPusher { @Resource private IConfigCenter iConfigCenter; + /** + * 将推送key到etcd的任务都丢到这个线程池里 + */ + private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(128); @Override public void push(HotKeyModel model) { - //推送到etcd，供dashboard监听入库 - iConfigCenter.putAndGrant(HotKeyPathTool.keyRecordPath(model), UUID.randomUUID().toString(), - KeyRuleHolder.getRuleByAppAndKey(model).getDuration()); + //2020-8-25日，新增线程池写入etcd的逻辑。本地测试发现8线程写入etcd，每秒最多500个key-value，128线程能写2500个以上 + EXECUTOR_SERVICE.submit(() -> { + //推送到etcd，供dashboard监听入库。 + iConfigCenter.putAndGrant(HotKeyPathTool.keyRecordPath(model), UUID.randomUUID().toString(), + KeyRuleHolder.getRuleByAppAndKey(model).getDuration()); + }); + } @Override -- Gitee From 2d7cab7715f1984a0811b3ddcbb4ee8a01f14376 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 15:33:09 +0800 Subject: [PATCH 13/59] =?UTF-8?q?worker=E5=90=91dashboard=E7=9B=B4?= =?UTF-8?q?=E6=8E=A8=E7=83=ADkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/jd/platform/hotkey/client/ClientStarter.java | 9 +-------- .../main/java/com/jd/platform/hotkey/client/Context.java | 2 +- .../com/jd/platform/hotkey/client/etcd/EtcdStarter.java | 2 +- 3 files changed, 3 insertions(+), 10 deletions(-) diff --git a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java index 1102d8a..27474e7 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.client; -import com.jd.platform.hotkey.client.cache.LocalCache; import com.jd.platform.hotkey.client.callback.ReceiveNewKeySubscribe; import com.jd.platform.hotkey.client.core.eventbus.EventBusCenter; import com.jd.platform.hotkey.client.core.key.PushSchedulerStarter; @@ -40,9 +39,8 @@ public class ClientStarter { public static class Builder { private String appName; private String etcdServer; - private LocalCache localCache; private Long pushPeriod; - private int caffeineSize = 50000; + private int caffeineSize = 200000; public Builder() { } @@ -65,11 +63,6 @@ public class ClientStarter { return this; } - public Builder setLocalCache(LocalCache localCache) { - this.localCache = localCache; - return this; - } - public Builder setPushPeriod(Long pushPeriod) { this.pushPeriod = pushPeriod; return this; diff --git a/client/src/main/java/com/jd/platform/hotkey/client/Context.java b/client/src/main/java/com/jd/platform/hotkey/client/Context.java index f0455a2..fea4106 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/Context.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/Context.java @@ -12,5 +12,5 @@ public class Context { */ public static boolean NEED_RECONNECT = true; - public static int CAFFEINE_SIZE = 50000; + public static int CAFFEINE_SIZE; } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java index 4c0b1b4..216160a 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/etcd/EtcdStarter.java @@ -241,7 +241,7 @@ public class EtcdStarter { JdLogger.info(getClass(), "trying to connect to etcd and fetch rule info"); boolean success = fetchRuleFromEtcd(); if (success) { - //拉取已经存在的热key + //拉取已存在的热key fetchExistHotKey(); scheduledExecutorService.shutdown(); -- Gitee From 38c88d7d82b7d51b8623bf6aea39a0e0ed3f6710 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 15:35:38 +0800 Subject: [PATCH 14/59] =?UTF-8?q?worker=E5=90=91dashboard=E7=9B=B4?= =?UTF-8?q?=E6=8E=A8=E7=83=ADkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/configcenter/ConfigConstant.java | 14 ++++++++ .../configcenter/etcd/JdEtcdClient.java | 1 - .../hotkey/common/model/HotKeyModel.java | 7 ++-- .../common/model/typeenum/MessageType.java | 3 +- .../hotkey/common/tool/flush/FlushUtil.java | 34 +++++++++++++++++++ 5 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index e88389d..2a7eff0 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -13,6 +13,10 @@ public interface ConfigConstant { * 所有的workers，存这里 */ String workersPath = "/jd/workers/"; + /** + * dashboard的ip存这里 + */ + String dashboardPath = "/jd/dashboard/"; /** * 该app所有的workers地址的path。需要手工分配，默认每个app都用所有的worker */ @@ -54,4 +58,14 @@ public interface ConfigConstant { * 存放客户端hotKey访问次数和总访问次数的path */ String keyHitCountPath = "/jd/keyHitCount/"; + + /** + * 清理历史数据的配置的path + * time unit : day + */ + String clearCfgPath = "/jd/clearCfg/"; + /** + * 控制台启动的netty端口 + */ + int dashboardPort = 11112; } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java index 06e85e9..fb64d9e 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java @@ -3,7 +3,6 @@ package com.jd.platform.hotkey.common.configcenter.etcd; import cn.hutool.core.collection.CollectionUtil; import com.google.protobuf.ByteString; import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.api.Kv; import com.ibm.etcd.api.LeaseGrantResponse; import com.ibm.etcd.api.RangeResponse; import com.ibm.etcd.client.KvStoreClient; diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java index 0451fce..11de0cc 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyModel.java @@ -23,11 +23,7 @@ public class HotKeyModel extends BaseModel { @Override public String toString() { - return "HotKeyModel{" + - "appName='" + appName + '\'' + - ", keyType=" + keyType + - ", remove=" + remove + - '}'; + return "appName:" + appName + "-key=" + getKey(); } public boolean isRemove() { @@ -53,4 +49,5 @@ public class HotKeyModel extends BaseModel { public void setKeyType(KeyType keyType) { this.keyType = keyType; } + } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java index cd3f983..5b07624 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java @@ -8,7 +8,8 @@ public enum MessageType { APP_NAME((byte) 1), REQUEST_NEW_KEY((byte) 2), RESPONSE_NEW_KEY((byte) 3), - REQUEST_HIT_COUNT((byte) 7), + REQUEST_HIT_COUNT((byte) 7), //命中率 + REQUEST_HOT_KEY((byte) 8), //热key，worker->dashboard PING((byte) 4), PONG((byte) 5), EMPTY((byte) 6); diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java new file mode 100644 index 0000000..83e74c1 --- /dev/null +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java @@ -0,0 +1,34 @@ +package com.jd.platform.hotkey.common.tool.flush; + +import io.netty.buffer.ByteBuf; +import io.netty.channel.ChannelHandlerContext; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author wuweifeng wrote on 2019-12-11 + * @version 1.0 + */ +public class FlushUtil { + private static Logger logger = LoggerFactory.getLogger("flushUtil"); + + /** + * 往channel里输出消息 + */ + public static void flush(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf) { +// if (channelHandlerContext.channel().isWritable()) { +// channelHandlerContext.channel().writeAndFlush(byteBuf).addListener(future -> { +// if (!future.isSuccess()) { +// logger.warn("flush error " + future.cause().getMessage()); +// } +// }); +// } else { + try { + //同步发送 + channelHandlerContext.channel().writeAndFlush(byteBuf).sync(); + } catch (InterruptedException e) { + logger.error("flush error " + e.getMessage()); + } +// } + } +} -- Gitee From b15f77736aab9d8cbd7997699d0c6dcc87350844 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 15:52:43 +0800 Subject: [PATCH 15/59] =?UTF-8?q?worker=E5=90=91dashboard=E7=9B=B4?= =?UTF-8?q?=E6=8E=A8=E7=83=ADkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/common/coder/Codec.java | 30 -- .../hotkey/common/coder/MessageDecoder.java | 53 ---- .../hotkey/common/coder/MessageEncoder.java | 47 --- .../hotkey/common/coder/NettyCodec.java | 36 --- .../hotkey/common/model/MsgBuilder.java | 23 +- .../dashboard/DashboardApplication.java | 19 +- .../dashboard/cache/CaffeineBuilder.java | 30 ++ .../dashboard/common/domain/Constant.java | 6 + .../dashboard/common/domain/EventWrapper.java | 95 ------ .../dashboard/common/domain/IRecord.java | 27 ++ .../hotkey/dashboard/common/domain/Page.java | 18 +- .../common/domain/vo/ClearCfgVo.java | 47 +++ .../dashboard/common/monitor/DataHandler.java | 120 ++++---- .../dashboard/common/monitor/EtcdMonitor.java | 256 ----------------- .../dashboard/controller/ClearController.java | 62 ++++ .../dashboard/controller/KeyController.java | 10 +- .../hotkey/dashboard/etcd/EtcdRegister.java | 63 ++++ .../dashboard/mapper/KeyRecordMapper.java | 4 +- .../dashboard/mapper/KeyTimelyMapper.java | 31 -- .../dashboard/mapper/StatisticsMapper.java | 4 +- .../dashboard/mapper/SummaryMapper.java | 4 +- .../hotkey/dashboard/model/KeyRecord.java | 8 +- .../hotkey/dashboard/model/KeyTimely.java | 25 +- .../dashboard/netty/HotKeyReceiver.java | 175 +++++++++++ .../hotkey/dashboard/netty/NodesServer.java | 78 +++++ .../dashboard/netty/NodesServerHandler.java | 66 +++++ .../dashboard/service/ClearService.java | 23 ++ .../hotkey/dashboard/service/KeyService.java | 7 +- .../service/impl/ClearServiceImpl.java | 85 ++++++ .../service/impl/KeyServiceImpl.java | 271 ++++-------------- .../service/impl/UserServiceImpl.java | 2 +- .../hotkey/dashboard/util/PageUtil.java | 36 +++ .../hotkey/dashboard/util/RuleUtil.java | 4 +- .../dashboard/mapper/KeyRecordMapper.xml | 6 + .../dashboard/mapper/KeyTimelyMapper.xml | 207 ------------- .../dashboard/mapper/StatisticsMapper.xml | 7 +- .../hotkey/dashboard/mapper/SummaryMapper.xml | 7 + .../admin/assets/js/bootstrap/js/base_list.js | 17 +- .../static/admin/common/js/clearcfg-edit.js | 25 ++ .../templates/admin/changeLog/list.html | 5 + .../resources/templates/admin/clear/edit.html | 69 +++++ .../resources/templates/admin/clear/list.html | 157 ++++++++++ .../templates/admin/common/html/leftMenu.html | 7 +- .../resources/templates/admin/key/list.html | 38 +-- .../templates/admin/key/listmaxhot.html | 4 +- .../templates/admin/key/listtimely.html | 2 +- .../templates/admin/rule/listhitcount.html | 43 +-- .../hotkey/worker/config/CodecConfig.java | 20 -- 48 files changed, 1216 insertions(+), 1163 deletions(-) delete mode 100755 common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java delete mode 100644 common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java delete mode 100644 common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java delete mode 100755 common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java delete mode 100644 dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml create mode 100644 dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js create mode 100644 dashboard/src/main/resources/templates/admin/clear/edit.html create mode 100644 dashboard/src/main/resources/templates/admin/clear/list.html delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java deleted file mode 100755 index c61805b..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/Codec.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jd.platform.hotkey.common.coder; - -import io.netty.channel.ChannelHandler; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -public interface Codec { - - ChannelHandler newEncoder(); - - ChannelHandler newDecoder(); -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java deleted file mode 100644 index 24cb0ed..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageDecoder.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.jd.platform.hotkey.common.coder; - -import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.Constant; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.ByteToMessageDecoder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.nio.charset.Charset; -import java.util.List; - -/** - * @author wuweifeng wrote on 2020-01-06 - * @version 1.0 - */ -public class MessageDecoder extends ByteToMessageDecoder { - private Logger logger = LoggerFactory.getLogger(getClass()); - - @Override - protected void decode(ChannelHandlerContext ctx, ByteBuf byteBuf, List out) { - if (byteBuf.readableBytes() < 10 || byteBuf.readableBytes() > Constant.MAX_LENGTH) { - logger.warn("数据包不正确，当前包大小为：" + byteBuf.readableBytes()); - return; - } - - byteBuf.markReaderIndex(); - - HotKeyMsg message = new HotKeyMsg(); - - int magicNumber = byteBuf.readInt(); - if (Constant.MAGIC_NUMBER != magicNumber) { - logger.warn("MAGIC_NUMBER不正确:" + magicNumber); - return; - } - - message.setMagicNumber(byteBuf.readInt()); // 读取魔数 - - MessageType messageType = MessageType.get(byteBuf.readByte()); - if (messageType == null) { - logger.error("messageType is null , byteBuf readByte = " + byteBuf.readByte()); - return; - } - message.setMessageType(messageType); // 读取当前的消息类型 - - int bodyLength = byteBuf.readInt(); // 读取消息体长度和数据 - CharSequence body = byteBuf.readCharSequence(bodyLength, Charset.defaultCharset()); - message.setBody(body.toString()); - out.add(message); - } -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java deleted file mode 100644 index acf2162..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/MessageEncoder.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.jd.platform.hotkey.common.coder; - -import com.jd.platform.hotkey.common.model.HotKeyMsg; -import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.Constant; -import io.netty.buffer.ByteBuf; -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.MessageToByteEncoder; - -import java.nio.charset.Charset; - -/** - * @author wuweifeng wrote on 2020-01-06 - * @version 1.0 - */ -public class MessageEncoder extends MessageToByteEncoder { - - @Override - protected void encode(ChannelHandlerContext ctx, HotKeyMsg message, ByteBuf out) { - MessageType messageType = message.getMessageType(); - //非法类型 - if (MessageType.get(messageType.getType()) == null) { - return; - } - // 这里会判断消息类型是不是EMPTY类型，如果是EMPTY类型，则表示当前消息不需要写入到管道中 - if (messageType == MessageType.EMPTY) { - return; - } - //4个byte - out.writeInt(Constant.MAGIC_NUMBER); // 写入当前的魔数 - //1个byte - out.writeByte(message.getMessageType().getType()); // 写入当前消息的类型 - - if (null == message.getBody()) { - out.writeInt(0); // 如果消息体为空，则写入0，表示消息体长度为0 - } else { - System.out.println(message.getBody().length()); - out.writeInt(message.getBody().length()); - out.writeCharSequence(message.getBody(), Charset.defaultCharset()); - } - out.writeBytes(Constant.DELIMITER.getBytes()); - } - - public static void main(String[] args) { - System.out.println(Constant.MAGIC_NUMBER + ""); - } -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java deleted file mode 100755 index 1aeac45..0000000 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/NettyCodec.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.jd.platform.hotkey.common.coder; - -import io.netty.channel.ChannelHandler; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -public class NettyCodec implements Codec { - - @Override - public ChannelHandler newEncoder() { - return new MessageEncoder(); - } - - @Override - public ChannelHandler newDecoder() { - return new MessageDecoder(); - } -} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/MsgBuilder.java b/common/src/main/java/com/jd/platform/hotkey/common/model/MsgBuilder.java index fc76264..f4630a4 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/MsgBuilder.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/MsgBuilder.java @@ -1,22 +1,21 @@ package com.jd.platform.hotkey.common.model; +import com.jd.platform.hotkey.common.tool.Constant; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + /** * @author wuweifeng * @version 1.0 * @date 2020-04-22 */ public class MsgBuilder { -// public static ByteBuf buildByteBuf(String msg) { -// return Unpooled.copiedBuffer((msg + Constant.DELIMITER).getBytes()); -// } + public static ByteBuf buildByteBuf(String msg) { + return Unpooled.copiedBuffer((msg + Constant.DELIMITER).getBytes()); + } -// public static ByteBuf buildByteBuf(HotKeyMsg hotKeyMsg) { -// byte[] bytes = ProtostuffUtils.serialize(hotKeyMsg); -// byte[] delimiter = Constant.DELIMITER.getBytes(); -// -// byte[] bt3 = new byte[bytes.length + delimiter.length]; -// System.arraycopy(bytes, 0, bt3, 0, bytes.length); -// System.arraycopy(delimiter, 0, bt3, bytes.length, delimiter.length); -// return Unpooled.copiedBuffer(bt3); -// } + public static ByteBuf buildByteBuf(HotKeyMsg hotKeyMsg) { + return Unpooled.copiedBuffer((FastJsonUtils.convertObjectToJSON(hotKeyMsg) + Constant.DELIMITER).getBytes()); + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 2b361c6..3070702 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -1,12 +1,6 @@ package com.jd.platform.hotkey.dashboard; -import com.alibaba.fastjson.JSON; -import com.jd.platform.hotkey.common.configcenter.ConfigConstant; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.tool.IpUtils; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; -import com.jd.platform.hotkey.dashboard.mapper.RulesMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; @@ -15,21 +9,14 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; -import javax.annotation.Resource; -import java.util.HashMap; -import java.util.Map; - @EnableAsync @EnableScheduling @SpringBootApplication public class DashboardApplication implements CommandLineRunner { - private Logger logger = LoggerFactory.getLogger(getClass()); - @Resource - private KeyTimelyMapper timelyMapper; public static void main(String[] args) { try { @@ -37,13 +24,11 @@ public class DashboardApplication implements CommandLineRunner { }catch (Exception e){ e.printStackTrace(); } - } + @Override - public void run(String... args) { - int row = timelyMapper.clear(); - logger.info("clear db timely hotKey, effect row : {}",row); + public void run(String... args) { } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java new file mode 100644 index 0000000..63612e8 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/cache/CaffeineBuilder.java @@ -0,0 +1,30 @@ +package com.jd.platform.hotkey.dashboard.cache; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +import java.util.concurrent.TimeUnit; + +/** + * @author wuweifeng wrote on 2019-12-12 + * @version 1.0 + */ +public class CaffeineBuilder { + + public static Cache cache(int duration) { + return cache(128, 1000000, duration); + } + + + /** + * 构建所有来的要缓存的key getCache + */ + public static Cache cache(int minSize, int maxSize, int expireSeconds) { + return Caffeine.newBuilder() + .initialCapacity(minSize)//初始大小 + .maximumSize(maxSize)//最大数量 + .expireAfterWrite(expireSeconds, TimeUnit.SECONDS)//过期时间 + .build(); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java index b11fa73..952aa2a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java @@ -35,6 +35,9 @@ public class Constant { public static final String WORKER_VIEW = "节点信息"; + public static final String CLEAR_VIEW = "数据清理"; + + public static final int MAX_DAY_RANGE = 3; public static final int VERSION = 1; @@ -55,6 +58,9 @@ public class Constant { public static final String INFO = "info"; + public static final String THIRTY_DAY = "30"; + + public static final List HEAD = new ArrayList<>(); static { HEAD.add("热点key"); HEAD.add("次数"); HEAD.add("所属APP");} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java deleted file mode 100644 index 04d4a83..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/EventWrapper.java +++ /dev/null @@ -1,95 +0,0 @@ -package com.jd.platform.hotkey.dashboard.common.domain; - -import com.ibm.etcd.api.Event; - -import java.io.Serializable; -import java.util.Date; - -/** - * 包装类，用于保存事件最准确的时间 - */ -public class EventWrapper implements Serializable { - - private String key; - - private String value; - - private Event.EventType eventType; - - private Date date; - - private Long ttl; - - private long version; - - private String uuid; - - @Override - public String toString() { - return "EventWrapper{" + - "key='" + key + '\'' + - ", value='" + value + '\'' + - ", eventType=" + eventType + - ", date=" + date + - ", ttl=" + ttl + - ", version=" + version + - ", uuid='" + uuid + '\'' + - '}'; - } - - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - - public long getVersion() { - return version; - } - - public void setVersion(long version) { - this.version = version; - } - - public String getKey() { - return key; - } - - public void setKey(String key) { - this.key = key; - } - - public String getValue() { - return value; - } - - public void setValue(String value) { - this.value = value; - } - - public Event.EventType getEventType() { - return eventType; - } - - public void setEventType(Event.EventType eventType) { - this.eventType = eventType; - } - - public void setDate(Date date) { - this.date = date; - } - - public void setTtl(Long ttl) { - this.ttl = ttl; - } - - public Date getDate() { - return date; - } - - public Long getTtl() { - return ttl; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java new file mode 100644 index 0000000..3585f73 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/IRecord.java @@ -0,0 +1,27 @@ +package com.jd.platform.hotkey.dashboard.common.domain; + +import java.util.Date; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-09-02 + */ +public interface IRecord { + /** + * appName + "/" + key + */ + String appNameKey(); + + /** + * 手工添加的是时间戳13位，worker传过来的是uuid + */ + String value(); + + /** + * 0插入，1删除 + */ + int type(); + + Date createTime(); +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java index 5f64c75..d10c3a5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Page.java @@ -7,9 +7,13 @@ import java.util.List; public class Page implements Serializable { private static final long serialVersionUID = 1L; - + /** + * 当前第几页 + */ private Integer page; - + /** + * 总共多少条 + */ private int total; private List rows; @@ -20,6 +24,7 @@ public class Page implements Serializable { this.rows = rows; } + public Integer getPage() { return page; } @@ -43,4 +48,13 @@ public class Page implements Serializable { public void setRows(List rows) { this.rows = rows; } + + @Override + public String toString() { + return "Page{" + + "page=" + page + + ", total=" + total + + ", rows=" + rows + + '}'; + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java new file mode 100644 index 0000000..14b872b --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java @@ -0,0 +1,47 @@ +package com.jd.platform.hotkey.dashboard.common.domain.vo; + +/** + * @ProjectName: hotkey + * @ClassName: ClearCfgVo + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:54 + */ +public class ClearCfgVo { + + private String app; + + private String ttl; + + private Long version; + + public ClearCfgVo(String app, String ttl, Long version) { + this.app = app; + this.ttl = ttl; + this.version = version; + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public String getTtl() { + return ttl; + } + + public void setTtl(String ttl) { + this.ttl = ttl; + } + + public Long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index b5b1a1e..bd3a5e0 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -1,20 +1,22 @@ package com.jd.platform.hotkey.dashboard.common.monitor; +import cn.hutool.core.collection.CollectionUtil; import com.alibaba.fastjson.JSON; -import com.ibm.etcd.api.Event; +import com.google.common.collect.Queues; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.dashboard.common.domain.Constant; -import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; +import com.jd.platform.hotkey.dashboard.common.domain.IRecord; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; +import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.KeyRecord; -import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; -import com.jd.platform.hotkey.dashboard.util.TwoTuple; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; @@ -22,10 +24,13 @@ import org.springframework.stereotype.Component; import javax.annotation.Resource; import java.time.LocalDateTime; +import java.util.ArrayList; import java.util.Date; import java.util.List; +import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; @Component public class DataHandler { @@ -35,21 +40,25 @@ public class DataHandler { @Resource private KeyRecordMapper keyRecordMapper; @Resource - private KeyTimelyMapper keyTimelyMapper; - @Resource private StatisticsMapper statisticsMapper; + @Resource + private SummaryMapper summaryMapper; + + @Resource + private IConfigCenter configCenter; + /** * 队列 */ - private BlockingQueue queue = new LinkedBlockingQueue<>(); + private BlockingQueue queue = new LinkedBlockingQueue<>(); /** * 入队 */ - public void offer(EventWrapper eventWrapper) { + public void offer(IRecord record) { try { - queue.put(eventWrapper); + queue.put(record); } catch (InterruptedException e) { e.printStackTrace(); } @@ -57,35 +66,25 @@ public class DataHandler { public void insertRecords() { while (true) { - TwoTuple twoTuple; try { - twoTuple = handHotKey(queue.take()); - if (twoTuple == null) { + List records = new ArrayList<>(); + Queues.drain(queue, records, 1000, 1, TimeUnit.SECONDS); + if (CollectionUtil.isEmpty(records)) { continue; } - } catch (Exception e) { - e.printStackTrace(); - log.error("handHotKey error ," + e.getCause()); - continue; - } - KeyRecord keyRecord = twoTuple.getSecond(); - KeyTimely keyTimely = twoTuple.getFirst(); - - if (keyTimely.getUuid() == null) { - keyTimelyMapper.deleteByKeyAndApp(keyTimely.getKey(), keyTimely.getAppName()); - } else { - try { - keyTimelyMapper.saveOrUpdate(keyTimely); - } catch (Exception e) { - log.info("insert timely error",e); + List keyRecordList = new ArrayList<>(); + for (IRecord iRecord : records) { + KeyRecord keyRecord = handHotKey(iRecord); + if (keyRecord != null) { + keyRecordList.add(keyRecord); + } } - } - if (keyRecord != null) { - //插入记录 - keyRecordMapper.insertSelective(keyRecord); - } + keyRecordMapper.batchInsert(keyRecordList); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } @@ -94,22 +93,19 @@ public class DataHandler { /** * 处理热点key和记录 */ - private TwoTuple handHotKey(EventWrapper eventWrapper) { - Date date = eventWrapper.getDate(); - long ttl = eventWrapper.getTtl(); - Event.EventType eventType = eventWrapper.getEventType(); - String appKey = eventWrapper.getKey(); - String value = eventWrapper.getValue(); + private KeyRecord handHotKey(IRecord record) { + Date date = record.createTime(); + String appKey = record.appNameKey(); + String value = record.value(); //appName+"/"+"key" String[] arr = appKey.split("/"); String appName = arr[0]; String key = arr[1]; - String uuid = eventWrapper.getUuid(); - int type = eventType.getNumber(); + String uuid = UUID.randomUUID().toString(); + int type = record.type(); //组建成对象，供累计后批量插入、删除 - TwoTuple timelyKeyRecordTwoTuple = new TwoTuple<>(); - if (eventType.equals(Event.EventType.PUT)) { + if (type == 0) { //如果是客户端删除时发出的put指令 if (com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE.equals(value)) { log.info("client remove key event : " + appKey); @@ -117,17 +113,14 @@ public class DataHandler { } //手工添加的是时间戳13位，worker传过来的是uuid String source = value.length() == 13 ? Constant.HAND : Constant.SYSTEM; - timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).val(value).appName(appName).duration(ttl).uuid(appKey).createTime(date).build()); String rule = RuleUtil.rule(appKey); - KeyRecord keyRecord = new KeyRecord(key, rule, appName, ttl, source, type, uuid, date); + KeyRecord keyRecord = new KeyRecord(key, rule, appName, 1, source, type, uuid, date); keyRecord.setRule(rule); - timelyKeyRecordTwoTuple.setSecond(keyRecord); - return timelyKeyRecordTwoTuple; - } else if (eventType.equals(Event.EventType.DELETE)) { - timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).appName(appName).build()); - return timelyKeyRecordTwoTuple; + return keyRecord; + } else { + //是删除 + return null; } - return timelyKeyRecordTwoTuple; } @@ -210,4 +203,29 @@ public class DataHandler { } + + /** + * 每天根据app的配置清理过期数据 + */ + @Scheduled(cron = "0 0 1 * * ?") + public void clearExpireData() { + try { + LocalDateTime now = LocalDateTime.now(); + List keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + for (KeyValue kv : keyValues) { + String key = kv.getKey().toStringUtf8(); + String ttl = kv.getValue().toStringUtf8(); + String app = key.replace(ConfigConstant.clearCfgPath, ""); + Date expireDate = DateUtil.ldtToDate(now.minusDays(Integer.parseInt(ttl))); + summaryMapper.clearExpireData(app, expireDate); + keyRecordMapper.clearExpireData(app, expireDate); + statisticsMapper.clearExpireData(app, expireDate); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java deleted file mode 100644 index 92fa472..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ /dev/null @@ -1,256 +0,0 @@ -package com.jd.platform.hotkey.dashboard.common.monitor; - -import cn.hutool.core.util.StrUtil; -import com.ibm.etcd.api.Event; -import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.client.kv.KvClient; -import com.jd.platform.hotkey.common.configcenter.ConfigConstant; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.rule.KeyRule; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import com.jd.platform.hotkey.dashboard.common.domain.Constant; -import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; -import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; -import com.jd.platform.hotkey.dashboard.model.Worker; -import com.jd.platform.hotkey.dashboard.service.WorkerService; -import com.jd.platform.hotkey.dashboard.util.CommonUtil; -import com.jd.platform.hotkey.dashboard.util.RuleUtil; -import io.grpc.StatusRuntimeException; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -/** - * @ProjectName: hotkey - * @ClassName: EtcdMonitor - * @Author: liyunfeng31 - * @Date: 2020/4/18 18:29 - */ -@SuppressWarnings("ALL") -@Component -public class EtcdMonitor { - - private static Logger log = LoggerFactory.getLogger(EtcdMonitor.class); - - @Resource - private IConfigCenter configCenter; - - @Resource - private WorkerService workerService; - - @Resource - private SummaryMapper summaryMapper; - - @Resource - private DataHandler dataHandler; - - - public static final ExecutorService threadPoolExecutor = Executors.newCachedThreadPool(); - - /** - * 监听新来的热key，该key的产生是来自于手工在控制台添加 - */ - public void watchHandOperationHotKey() { - threadPoolExecutor.submit(() -> { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyPath); - while (watchIterator.hasNext()) { - Event event = event(watchIterator); - EventWrapper eventWrapper = build(event); - - String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyPath, ""); - eventWrapper.setKey(appKey); - dataHandler.offer(eventWrapper); - } - }); - } - - /** - * 监听新来的热key，该key的产生是来自于worker集群推送过来的 - */ - public void watchHotKeyRecord() { - threadPoolExecutor.submit(() -> { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyRecordPath); - while (watchIterator.hasNext()) { - Event event = event(watchIterator); - EventWrapper eventWrapper = build(event); - - String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyRecordPath, ""); - eventWrapper.setKey(appKey); - - dataHandler.offer(eventWrapper); - } - }); - } - - private EventWrapper build(Event event) { - KeyValue kv = event.getKv(); - long ttl = configCenter.timeToLive(kv.getLease()); - String v = kv.getValue().toStringUtf8(); - Event.EventType eventType = event.getType(); - EventWrapper eventWrapper = new EventWrapper(); - eventWrapper.setValue(v); - eventWrapper.setDate(new Date()); - eventWrapper.setTtl(ttl); - eventWrapper.setVersion(kv.getVersion()); - eventWrapper.setEventType(eventType); - eventWrapper.setUuid(v); - return eventWrapper; - } - - - @PostConstruct - public void startWatch() { - //拉取rules - fetchRuleFromEtcd(); - - //规则拉取完毕后才能去开始入库 - insertRecords(); - - //开始监听热key产生 - watchHotKeyRecord(); - - watchHandOperationHotKey(); - - //监听rule变化 - watchRule(); - - watchWorkers(); - - //观察热key访问次数和总访问次数，并做统计 - watchHitCount(); - } - - private void insertRecords() { - threadPoolExecutor.submit(() -> { - dataHandler.insertRecords(); - }); - } - - /** - * 异步监听rule规则变化 - */ - private void watchRule() { - threadPoolExecutor.submit(() -> { - - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.rulePath); - //如果有新事件，即rule的变更，就重新拉取所有的信息 - while (watchIterator.hasNext()) { - //这句必须写，next会让他卡住，除非真的有新rule变更 - Event event = event(watchIterator); - - try { - log.info("---------watch rule change---------"); - //全量拉取rule信息 - fetchRuleFromEtcd(); - } catch (Exception e) { - log.error("watch rule err"); - } - } - - }); - } - - /** - * 启动后从etcd拉取所有rule - */ - private void fetchRuleFromEtcd() { - RuleUtil.init(); - try { - List ruleList = new ArrayList<>(); - - //从etcd获取rule - List keyValues = configCenter.getPrefix(ConfigConstant.rulePath); - - for (KeyValue keyValue : keyValues) { - try { - log.info(keyValue.getKey() + "---" + keyValue.getValue()); - String appName = keyValue.getKey().toStringUtf8().replace(ConfigConstant.rulePath, ""); - if (StrUtil.isEmpty(appName)) { - configCenter.delete(keyValue.getKey().toStringUtf8()); - continue; - } - String rulesStr = keyValue.getValue().toStringUtf8(); - RuleUtil.put(appName, FastJsonUtils.toList(rulesStr, KeyRule.class)); - } catch (Exception e) { - log.error("rule parse failure"); - } - - } - - } catch (StatusRuntimeException ex) { - //etcd连不上 - log.error("etcd connected fail. Check the etcd address!!!"); - } - - } - - private void watchWorkers() { - threadPoolExecutor.submit(() -> { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.workersPath); - while (watchIterator.hasNext()) { - Event event = event(watchIterator); - KeyValue kv = event.getKv(); - Event.EventType eventType = event.getType(); - String k = kv.getKey().toStringUtf8(); - String v = kv.getValue().toStringUtf8(); - long version = kv.getModRevision(); - String uuid = k + Constant.JOIN + version; - Worker worker = new Worker(k, v, uuid); - if (eventType.equals(Event.EventType.PUT)) { - workerService.insertWorkerBySys(worker); - } else if (eventType.equals(Event.EventType.DELETE)) { - worker.setState(0); - workerService.updateWorker(worker); - } - } - }); - } - - - - /** - * 监听热key访问次数和总访问次数 - */ - private void watchHitCount() { - threadPoolExecutor.submit(() -> { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.keyHitCountPath); - while (watchIterator.hasNext()) { - Event event = event(watchIterator); - KeyValue kv = event.getKv(); - Event.EventType eventType = event.getType(); - if (Event.EventType.DELETE.equals(eventType)) { - continue; - } - String k = kv.getKey().toStringUtf8(); - String v = kv.getValue().toStringUtf8(); - - try { - Map map = FastJsonUtils.stringToCollect(v); - for (String key : map.keySet()) { - int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map)); - } - } catch (Exception e) { - e.printStackTrace(); - } - - } - }); - } - - - private Event event(KvClient.WatchIterator watchIterator) { - return watchIterator.next().getEvents().get(0); - } - - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java new file mode 100644 index 0000000..0ce81f0 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java @@ -0,0 +1,62 @@ +package com.jd.platform.hotkey.dashboard.controller; + +import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.Result; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; +import com.jd.platform.hotkey.dashboard.service.ClearService; +import org.springframework.stereotype.Controller; +import org.springframework.ui.ModelMap; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; + +/** + * @ProjectName: hotkey + * @ClassName: ClearController + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:48 + */ +@Controller +@RequestMapping("/clear") +public class ClearController { + + + private String prefix = "admin/clear"; + + + @Resource + private ClearService clearService; + + @GetMapping("/view") + public String view(ModelMap modelMap){ + modelMap.put("title", Constant.CLEAR_VIEW); + return prefix + "/list"; + } + + + @PostMapping("/list") + @ResponseBody + public Page list(PageReq page, String app){ + PageInfo info = clearService.pageClearCfg(page, app); + return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); + } + + + @GetMapping("/edit/{app}") + public String edit(@PathVariable("app") String app, ModelMap modelMap){ + modelMap.put("clearCfg", clearService.selectClearCfg(app)); + return prefix + "/edit"; + } + + @PostMapping("/save") + @ResponseBody + public Result save(ClearCfgVo cfg){ + int b = clearService.saveClearCfg(cfg); + return b == 0 ? Result.fail():Result.success(); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java index 0b287a8..f3f362d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java @@ -79,8 +79,7 @@ public class KeyController extends BaseController { @PostMapping("/listTimely") @ResponseBody public Page listTimely(PageReq page, SearchReq searchReq){ - PageInfo info = keyService.pageKeyTimely(page, searchReq); - return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); + return keyService.pageKeyTimely(page, searchReq); } @@ -123,13 +122,6 @@ public class KeyController extends BaseController { } - @GetMapping("/edit/{id}") - public String edit(@PathVariable("id") Long id, ModelMap modelMap){ - modelMap.put("key", keyService.selectByPk(id)); - return prefix + "/edit"; - } - - @PostMapping("/edit") @ResponseBody public Result editSave(KeyTimely key) { diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java new file mode 100644 index 0000000..d0a44c5 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdRegister.java @@ -0,0 +1,63 @@ +package com.jd.platform.hotkey.dashboard.etcd; + +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.tool.IpUtils; +import com.jd.platform.hotkey.dashboard.netty.NodesServer; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +/** + * dashboard也注册到etcd，供各个worker连接 + * + * @author wuweifeng + * @version 1.0 + * @date 2020-08-28 + */ +@Component +public class EtcdRegister { + @Resource + private IConfigCenter configCenter; + @Resource + private NodesServer nodesServer; + + /** + * 每隔一会去check一下，自己还在不在etcd里 + */ + @PostConstruct + public void makeSureSelfOn() { + new Thread(() -> nodesServer.startNettyServer(ConfigConstant.dashboardPort)).start(); + + //开启上传worker信息 + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + scheduledExecutorService.scheduleAtFixedRate(() -> { + + try { + uploadSelfInfo(); + } catch (Exception e) { + //do nothing + } + + }, 3, 30, TimeUnit.SECONDS); + } + + private void uploadSelfInfo() { + configCenter.putAndGrant(buildKey(), buildValue(), 32); + } + + private String buildKey() { + String hostName = IpUtils.getHostName(); + return ConfigConstant.dashboardPath + hostName; + } + + private String buildValue() { + String ip = IpUtils.getIp(); + return ip + ":" + ConfigConstant.dashboardPort; + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java index 3ef46b3..e135ff6 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java @@ -1,11 +1,11 @@ package com.jd.platform.hotkey.dashboard.mapper; -import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.Statistics; import org.apache.ibatis.annotations.Mapper; +import java.util.Date; import java.util.List; /** @@ -25,4 +25,6 @@ public interface KeyRecordMapper { List maxHotKey(SearchReq req); List statisticsByRule(SearchReq req); + + int clearExpireData(String app, Date expireDate); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java deleted file mode 100644 index 3423da3..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jd.platform.hotkey.dashboard.mapper; - -import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.model.KeyTimely; -import org.apache.ibatis.annotations.Mapper; - -import java.util.List; - -@Mapper -public interface KeyTimelyMapper { - - int clear(); - - int deleteByKeyAndApp(String key,String appName); - - int insertSelective(KeyTimely key); - - KeyTimely selectByPrimaryKey(Long id); - - KeyTimely selectByKey(String key); - - int updateByKey(KeyTimely key); - - List listKeyTimely(SearchReq param); - - int batchInsert(List list); - - void batchDeleted(List deleteList); - - void saveOrUpdate(KeyTimely keyTimely); -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java index dabdfc5..daf848e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java @@ -4,6 +4,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.Statistics; import org.apache.ibatis.annotations.Mapper; +import java.util.Date; import java.util.List; /** @@ -42,7 +43,6 @@ public interface StatisticsMapper { /** * 清理 - * @param type type */ - void clear(int type); + int clearExpireData(String app, Date expireDate); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java index 4685e76..0305c4c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java @@ -2,10 +2,10 @@ package com.jd.platform.hotkey.dashboard.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; -import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.model.Summary; import org.apache.ibatis.annotations.Mapper; +import java.util.Date; import java.util.List; /** @@ -22,4 +22,6 @@ public interface SummaryMapper { int saveOrUpdate(Summary records); List listRuleHitCount(SearchReq req); + + int clearExpireData(String app, Date expireDate); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java index 97f0b4b..e8fdf0f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRecord.java @@ -16,7 +16,7 @@ public class KeyRecord implements Serializable { /** * 缓存时间 */ - private Long duration; + private int duration; /** * 来源： SYSTEM 系统探测；USERNAME创建人 @@ -43,7 +43,7 @@ public class KeyRecord implements Serializable { public KeyRecord() { } - public KeyRecord(String key,String val, String appName, Long duration, + public KeyRecord(String key,String val, String appName, int duration, String source, Integer type,String uuid, Date createTime) { this.key = key; this.val = val; @@ -103,11 +103,11 @@ public class KeyRecord implements Serializable { this.val = val; } - public Long getDuration() { + public int getDuration() { return duration; } - public void setDuration(Long duration) { + public void setDuration(int duration) { this.duration = duration; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java index 6efd7c5..67b9d0a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java @@ -22,7 +22,6 @@ public class KeyTimely implements Serializable { private Date createTime; - private String uuid; /** * 该rule的描述 */ @@ -44,12 +43,25 @@ public class KeyTimely implements Serializable { public KeyTimely(String key, String val, String appName, Long duration, String uuid, Date createTime) { this.key = key; this.val = val; - this.uuid = uuid; this.appName = appName; this.duration = duration; this.createTime = createTime; } + @Override + public String toString() { + return "KeyTimely{" + + "id=" + id + + ", key='" + key + '\'' + + ", appName='" + appName + '\'' + + ", val='" + val + '\'' + + ", duration=" + duration + + ", createTime=" + createTime + + ", ruleDesc='" + ruleDesc + '\'' + + ", updater='" + updater + '\'' + + '}'; + } + public String getRuleDesc() { return ruleDesc; } @@ -106,14 +118,6 @@ public class KeyTimely implements Serializable { this.createTime = createTime; } - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - public String getUpdater() { return updater; } @@ -186,7 +190,6 @@ public class KeyTimely implements Serializable { keyTimely.setVal(val); keyTimely.setDuration(duration); keyTimely.setCreateTime(createTime); - keyTimely.setUuid(uuid); keyTimely.setRuleDesc(ruleDesc); keyTimely.setUpdater(updater); return keyTimely; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java new file mode 100644 index 0000000..88057c9 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java @@ -0,0 +1,175 @@ +package com.jd.platform.hotkey.dashboard.netty; + +import cn.hutool.core.util.StrUtil; +import com.github.benmanes.caffeine.cache.Cache; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.rule.KeyRule; +import com.jd.platform.hotkey.dashboard.cache.CaffeineBuilder; +import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; +import com.jd.platform.hotkey.dashboard.model.KeyTimely; +import com.jd.platform.hotkey.dashboard.util.RuleUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.stream.Collectors; + +/** + * 此处存储所有发来的热key，统一处理入库 + * + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +public class HotKeyReceiver { + + /** + * 发来的热key集中营 + */ + private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); + /** + * 存储实时hotkey，供界面查询实时热key + */ + private static Map> aliveKeyStore = new ConcurrentHashMap<>(); + + private static Logger logger = LoggerFactory.getLogger("HotKeyReceiver"); + + /** + * netty收到的先存这里 + */ + public static void push(HotKeyModel model) { + hotKeyStoreQueue.offer(model); + } + + public static HotKeyModel take() { + try { + return hotKeyStoreQueue.take(); + } catch (InterruptedException e) { + e.printStackTrace(); + return null; + } + } + + /** + * 将热key存入本地缓存，设置过期时间 + */ + public static void put(HotKeyModel hotKeyModel) { + String appNameKey = hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + logger.error("rule is null, hotkeyModel " + hotKeyModel.getAppName() + "-" + hotKeyModel.getKey()); + return; + } + Cache cache = aliveKeyStore.computeIfAbsent(keyRule.getDuration(), s -> CaffeineBuilder.cache(keyRule.getDuration())); + cache.put(appNameKey, hotKeyModel); + } + + /** + * 展示当前所有的实时热key + */ + public static List list(SearchReq searchReq) { + List timelyList = new ArrayList<>(); + + long now = System.currentTimeMillis(); + + for (Integer duration : aliveKeyStore.keySet()) { + Cache cache = aliveKeyStore.get(duration); + ConcurrentMap concurrentHashMap = cache.asMap(); + for (Map.Entry entry : concurrentHashMap.entrySet()) { + KeyTimely keyTimely = parse((HotKeyModel) entry.getValue(), now); + if (keyTimely == null) { + continue; + } + timelyList.add(keyTimely); + } + } + + if (searchReq != null) { + if (StrUtil.isNotEmpty(searchReq.getApp())) { + timelyList = timelyList.parallelStream().filter(keyTimely -> searchReq.getApp().equals(keyTimely.getAppName())).collect(Collectors.toList()); + } + if (StrUtil.isNotEmpty(searchReq.getKey())) { + timelyList = timelyList.parallelStream().filter(keyTimely -> keyTimely.getKey().startsWith(searchReq.getKey())).collect(Collectors.toList()); + } + + } + + timelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); + + return timelyList; + } + + /** + * 将hotkeyModel变成前端需要的对象 + */ + private static KeyTimely parse(HotKeyModel hotKeyModel, long now) { + String appNameKey = hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + return null; + } + long remainTime = keyRule.getDuration() * 1000 - (now - hotKeyModel.getCreateTime()); + return KeyTimely.aKeyTimely() + .key(hotKeyModel.getKey()) + .val(UUID.randomUUID().toString()) + .appName(hotKeyModel.getAppName()) + .duration(remainTime / 1000) + .ruleDesc(RuleUtil.ruleDesc(appNameKey)) + .createTime(new Date(hotKeyModel.getCreateTime())).build(); + } + +// public static void main(String[] args) { +// List keyTimelyList = new ArrayList<>(); +// for (int i = 0; i < 24; i++) { +// KeyTimely keyTimely = new KeyTimely(); +// keyTimely.setKey(i + ""); +// keyTimely.setCreateTime(new Date()); +// try { +// Thread.sleep(10); +// } catch (InterruptedException e) { +// e.printStackTrace(); +// } +// keyTimelyList.add(keyTimely); +// } +// keyTimelyList.sort(Comparator.comparing(KeyTimely::getCreateTime).reversed()); +// +// Page page =PageUtil.pagination(keyTimelyList, 10, 3); +// System.out.println(page.getTotal()); +// System.out.println(page.getPage()); +// System.out.println(page.getRows()); +// } + + + + + + /** + * 删除实时热key + */ + public static boolean delete(String appNameKey) { + KeyRule keyRule = RuleUtil.findByKey(appNameKey); + if (keyRule == null) { + return false; + } + Cache cache = aliveKeyStore.get(keyRule.getDuration()); + if (cache == null) { + return false; + } + cache.invalidate(appNameKey); + return true; + } + + /** + * 定时清理caffeine + */ + public static void cleanUpCaffeine() { + for (Integer duration : aliveKeyStore.keySet()) { + Cache cache = aliveKeyStore.get(duration); + cache.cleanUp(); + } + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java new file mode 100644 index 0000000..a3dc7ee --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServer.java @@ -0,0 +1,78 @@ +package com.jd.platform.hotkey.dashboard.netty; + +import com.jd.platform.hotkey.common.tool.Constant; +import io.netty.bootstrap.ServerBootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.nio.NioServerSocketChannel; +import io.netty.handler.codec.DelimiterBasedFrameDecoder; +import io.netty.handler.codec.string.StringDecoder; +import io.netty.handler.logging.LogLevel; +import io.netty.handler.logging.LoggingHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +import java.util.concurrent.TimeUnit; + +/** + * 该server用于给各个worker实例连接用。 + * + * @author wuweifeng wrote on 2019-11-05. + */ +@Component +public class NodesServer { + private Logger logger = LoggerFactory.getLogger(getClass()); + + public void startNettyServer(int port) { + //boss单线程 + EventLoopGroup bossGroup = new NioEventLoopGroup(1); + EventLoopGroup workerGroup = new NioEventLoopGroup(4); + try { + ServerBootstrap bootstrap = new ServerBootstrap(); + bootstrap.group(bossGroup, workerGroup) + .channel(NioServerSocketChannel.class) + .handler(new LoggingHandler(LogLevel.INFO)) + .option(ChannelOption.SO_BACKLOG, 1024) + //保持长连接 + .childOption(ChannelOption.SO_KEEPALIVE, true) + //出来网络io事件，如记录日志、对消息编解码等 + .childHandler(new ChildChannelHandler()); + //绑定端口，同步等待成功 + ChannelFuture future = bootstrap.bind(port).sync(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + bossGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + workerGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + })); + //等待服务器监听端口关闭 + future.channel().closeFuture().sync(); + } catch (Exception e) { + e.printStackTrace(); + logger.error("dashboard netty server start failure"); + } finally { + //优雅退出，释放线程池资源 + bossGroup.shutdownGracefully(); + workerGroup.shutdownGracefully(); + } + } + + /** + * handler类 + */ + private class ChildChannelHandler extends ChannelInitializer { + + @Override + protected void initChannel(Channel ch) { + NodesServerHandler serverHandler = new NodesServerHandler(); + + ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); + ch.pipeline() + .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) + .addLast(new StringDecoder()) + .addLast(serverHandler); + } + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java new file mode 100644 index 0000000..9650c2f --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/NodesServerHandler.java @@ -0,0 +1,66 @@ +package com.jd.platform.hotkey.dashboard.netty; + +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.MsgBuilder; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.common.tool.flush.FlushUtil; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.util.StringUtils; + +import java.util.List; + +import static com.jd.platform.hotkey.common.tool.Constant.PONG; + +/** + * 这里处理所有netty事件。 + * + * @author wuweifeng wrote on 2019-11-05. + */ +public class NodesServerHandler extends SimpleChannelInboundHandler { + + private Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + protected void channelRead0(ChannelHandlerContext ctx, String message) { + if (StringUtils.isEmpty(message)) { + return; + } + try { + HotKeyMsg msg = FastJsonUtils.toBean(message, HotKeyMsg.class); + if (MessageType.PING == msg.getMessageType()) { + String hotMsg = FastJsonUtils.convertObjectToJSON(new HotKeyMsg(MessageType.PONG, PONG)); + FlushUtil.flush(ctx, MsgBuilder.buildByteBuf(hotMsg)); + } else if (MessageType.REQUEST_HOT_KEY == msg.getMessageType()) { + List list = FastJsonUtils.toList(msg.getBody(), HotKeyModel.class); + for (HotKeyModel hotKeyModel : list) { + HotKeyReceiver.push(hotKeyModel); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + + @Override + public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) throws Exception { + logger.error("some thing is error , " + cause.getMessage()); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) throws Exception { + super.channelActive(ctx); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + ctx.close(); + super.channelInactive(ctx); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java new file mode 100644 index 0000000..5266ef3 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java @@ -0,0 +1,23 @@ +package com.jd.platform.hotkey.dashboard.service; + +import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; + +/** + * @ProjectName: hotkey + * @ClassName: ClearService + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:51 + */ +public interface ClearService { + + + PageInfo pageClearCfg(PageReq page, String app); + + ClearCfgVo selectClearCfg(String app); + + int saveClearCfg(ClearCfgVo cfg); + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java index 3dfcf18..558e814 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.dashboard.service; import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; @@ -29,11 +30,7 @@ public interface KeyService { int delKeyByUser(KeyTimely keyTimely); - KeyTimely selectByKey(String key); - - KeyTimely selectByPk(Long key); - - PageInfo pageKeyTimely(PageReq page, SearchReq param); + Page pageKeyTimely(PageReq page, SearchReq param); PageInfo pageMaxHot(PageReq page, SearchReq param); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java new file mode 100644 index 0000000..8227a95 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java @@ -0,0 +1,85 @@ +package com.jd.platform.hotkey.dashboard.service.impl; + +import com.github.pagehelper.PageInfo; +import com.github.pagehelper.util.StringUtil; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; +import com.jd.platform.hotkey.dashboard.service.ClearService; +import com.jd.platform.hotkey.dashboard.service.UserService; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; + +/** + * @ProjectName: hotkey + * @ClassName: ClearServiceImpl + * @Description: TODO(一句话描述该类的功能) + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:57 + */ +@Service +public class ClearServiceImpl implements ClearService { + + + @Resource + private IConfigCenter configCenter; + + @Resource + private UserService userService; + + + @Override + public PageInfo pageClearCfg(PageReq page, String app) { + List keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + if(CollectionUtils.isEmpty(keyValues)){ + List apps = userService.listApp(); + for (String ap : apps) { + configCenter.put(ConfigConstant.clearCfgPath + ap, Constant.THIRTY_DAY); + } + keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + } + List cfgVos = new ArrayList<>(); + for (KeyValue kv : keyValues) { + String v = kv.getValue().toStringUtf8(); + String key = kv.getKey().toStringUtf8(); + if(StringUtil.isEmpty(v)){ + configCenter.put(key, Constant.THIRTY_DAY); + continue; + } + long version = kv.getModRevision(); + String k = key.replace(ConfigConstant.clearCfgPath,""); + if(StringUtils.isEmpty(app)){ + cfgVos.add(new ClearCfgVo(k, v, version)); + }else{ + if(k.equals(app)){ + cfgVos.add(new ClearCfgVo(k, v, version)); + } + } + } + return new PageInfo<>(cfgVos); + } + + @Override + public ClearCfgVo selectClearCfg(String app) { + KeyValue kv = configCenter.getKv(ConfigConstant.clearCfgPath + app); + if(kv == null || kv.getValue() == null){ + configCenter.put(ConfigConstant.clearCfgPath + app, Constant.THIRTY_DAY); + return new ClearCfgVo(app, Constant.THIRTY_DAY, 0L); + } + String v = kv.getValue().toStringUtf8(); + return new ClearCfgVo(app,v,kv.getModRevision()); + } + + @Override + public int saveClearCfg(ClearCfgVo cfg) { + configCenter.put(ConfigConstant.clearCfgPath + cfg.getApp(), cfg.getTtl()); + return 1; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 88a4263..09dbd32 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -8,24 +8,29 @@ import com.github.pagehelper.PageInfo; import com.ibm.etcd.api.Event; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; import com.jd.platform.hotkey.dashboard.model.ChangeLog; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; +import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.service.KeyService; import com.jd.platform.hotkey.dashboard.service.RuleService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.DateUtil; +import com.jd.platform.hotkey.dashboard.util.PageUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; import javax.annotation.Resource; @@ -50,16 +55,17 @@ public class KeyServiceImpl implements KeyService { @Resource private KeyRecordMapper recordMapper; @Resource - private KeyTimelyMapper keyTimelyMapper; - @Resource private StatisticsMapper statisticsMapper; @Resource private RuleService ruleService; @Resource private ChangeLogMapper logMapper; + private Logger logger = LoggerFactory.getLogger(getClass()); + /** * 折线图 + * * @param req req * @return vo */ @@ -68,52 +74,48 @@ public class KeyServiceImpl implements KeyService { int type = req.getType(); String appReq = req.getApp(); // admin 全查 - if(StrUtil.isNotEmpty(appReq)){ + if (StrUtil.isNotEmpty(appReq)) { app = appReq; } req.setApp(null); LocalDateTime now = LocalDateTime.now(); req.setEndTime(req.getEndTime() == null ? DateUtil.ldtToDate(now) : req.getEndTime()); List rules = ruleService.listRules(null); - if(type == 4){ - LocalDateTime st = req.getStartTime() == null? now.minusMinutes(31) : DateUtil.dateToLdt(req.getStartTime()); + if (type == 4) { + LocalDateTime st = req.getStartTime() == null ? now.minusMinutes(31) : DateUtil.dateToLdt(req.getStartTime()); req.setStartTime(DateUtil.ldtToDate(st)); LocalDateTime et = DateUtil.dateToLdt(req.getEndTime()); - boolean longTime = Duration.between(st,et).toHours()>2; + boolean longTime = Duration.between(st, et).toHours() > 2; req.setType(longTime ? 6 : 5); List list = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(st,et,list,!longTime,rules,app); + return CommonUtil.processData(st, et, list, !longTime, rules, app); } - if(type == 5){ + if (type == 5) { LocalDateTime startTime = now.minusMinutes(31); req.setStartTime(DateUtil.ldtToDate(startTime)); List list = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(startTime,now,list,true,rules,app); - }else if(type == 6){ + return CommonUtil.processData(startTime, now, list, true, rules, app); + } else if (type == 6) { LocalDateTime startTime2 = now.minusHours(25); req.setStartTime(DateUtil.ldtToDate(startTime2)); List list2 = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(startTime2,now,list2,false,rules,app); - }else{ + return CommonUtil.processData(startTime2, now, list2, false, rules, app); + } else { LocalDateTime startTime3 = now.minusDays(7).minusHours(1); req.setStartTime(DateUtil.ldtToDate(startTime3)); req.setType(6); List list3 = statisticsMapper.listOrderByTime(req); - return CommonUtil.processData(startTime3,now,list3,false,rules,app); + return CommonUtil.processData(startTime3, now, list3, false, rules, app); } } @Override - public PageInfo pageKeyTimely(PageReq page, SearchReq param) { - PageHelper.startPage(page.getPageNum(), page.getPageSize()); - List listKey = keyTimelyMapper.listKeyTimely(param); - for (KeyTimely timely : listKey) { - timely.setKey(CommonUtil.keyName(timely.getKey())); - timely.setRuleDesc(RuleUtil.ruleDesc(timely.getAppName() + "/" + timely.getKey())); - } - return new PageInfo<>(listKey); + public Page pageKeyTimely(PageReq page, SearchReq param) { + List keyTimelies = HotKeyReceiver.list(param); + return PageUtil.pagination(keyTimelies, page.getPageSize(), page.getPageNum()-1); + } @Override @@ -143,12 +145,12 @@ public class KeyServiceImpl implements KeyService { Map keyDateMap = keyDateMap(statistics, hours); // 获取时间x轴 List list = new ArrayList<>(); - for (int i = hours; i >0 ; i--) { - LocalDateTime time = LocalDateTime.now().minusHours(i-1); + for (int i = hours; i > 0; i--) { + LocalDateTime time = LocalDateTime.now().minusHours(i - 1); int hour = time.getHour(); - list.add(hour+"时"); + list.add(hour + "时"); } - return new HotKeyLineChartVo(list,keyDateMap); + return new HotKeyLineChartVo(list, keyDateMap); } @@ -168,6 +170,13 @@ public class KeyServiceImpl implements KeyService { public int insertKeyByUser(KeyTimely key) { configCenter.putAndGrant(ConfigConstant.hotKeyPath + key.getAppName() + "/" + key.getKey(), System.currentTimeMillis() + "", key.getDuration()); + + //写入本地缓存，实时热key信息 + HotKeyModel hotKeyModel = new HotKeyModel(); + hotKeyModel.setCreateTime(System.currentTimeMillis()); + hotKeyModel.setAppName(key.getAppName()); + hotKeyModel.setKey(key.getKey()); + HotKeyReceiver.put(hotKeyModel); return logMapper.insertSelective(new ChangeLog(key.getAppName(), Constant.HOTKEY_CHANGE, "", key.getKey(), key.getUpdater(), SystemClock.now() + "")); } @@ -185,56 +194,46 @@ public class KeyServiceImpl implements KeyService { String[] arr = keyTimely.getKey().split("/"); //删除client监听目录的key String etcdKey = ConfigConstant.hotKeyPath + arr[0] + "/" + arr[1]; + + //删除caffeine里的实时key + HotKeyReceiver.delete(arr[0] + "/" + arr[1]); + if (configCenter.get(etcdKey) == null) { //如果手工目录也就是client监听的目录里没有该key，那么就往里面放一个，然后再删掉它，这样client才能监听到删除事件 configCenter.putAndGrant(etcdKey, com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE, 1); } configCenter.delete(etcdKey); - //也删除Record目录下的该key，因为不确定要删的key到底在哪 - String recordKey = ConfigConstant.hotKeyRecordPath + arr[0] + "/" + arr[1]; - configCenter.delete(recordKey); - - KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0L, Constant.HAND, + KeyRecord keyRecord = new KeyRecord(arr[1], "", arr[0], 0, Constant.HAND, Event.EventType.DELETE_VALUE, UUID.randomUUID().toString(), new Date()); recordMapper.insertSelective(keyRecord); - return logMapper.insertSelective(new ChangeLog(arr[0], Constant.HOTKEY_CHANGE, arr[1],"", keyTimely.getUpdater(), SystemClock.now() + "")); + return logMapper.insertSelective(new ChangeLog(keyTimely.getKey(), Constant.HOTKEY_CHANGE, keyTimely.getKey(), "", keyTimely.getUpdater(), SystemClock.now() + "")); } - @Override - public KeyTimely selectByKey(String key) { - return keyTimelyMapper.selectByKey(key); - } - - @Override - public KeyTimely selectByPk(Long id) { - return keyTimelyMapper.selectByPrimaryKey(id); - } - - - - private Map keyDateMap(List statistics, int hours){ + private Map keyDateMap(List statistics, int hours) { Map map = new HashMap<>(10); Map> listMap = statistics.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); for (Map.Entry> m : listMap.entrySet()) { int start = DateUtil.preHoursInt(5); - map.put(m.getKey(),new int[hours]); + map.put(m.getKey(), new int[hours]); int[] data = map.get(m.getKey()); int tmp = 0; for (int i = 0; i < hours; i++) { Statistics st; try { st = m.getValue().get(tmp); - if(String.valueOf(start).endsWith("24")){ start = start + 77; } - if(start != st.getHours()){ + if (String.valueOf(start).endsWith("24")) { + start = start + 77; + } + if (start != st.getHours()) { data[i] = 0; - }else{ - tmp ++; + } else { + tmp++; data[i] = st.getCount(); } start++; - }catch (Exception e){ + } catch (Exception e) { data[i] = 0; } } @@ -243,9 +242,8 @@ public class KeyServiceImpl implements KeyService { } - private void checkParam(SearchReq req) { - if(req.getStartTime() == null || req.getEndTime() == null){ + if (req.getStartTime() == null || req.getEndTime() == null) { req.setStartTime(DateUtil.preTime(5)); req.setEndTime(new Date()); } @@ -255,173 +253,6 @@ public class KeyServiceImpl implements KeyService { }*/ } - - private static List statisticsList(){ - Random rd = new Random(); - List records = new ArrayList<>(); - // statisticsMapper.clear(1); - for (int i = 0; i < 35; i++) { - if(i== 2||i== 3||i== 4||i== 25||i== 12||i== 13||i== 14||i== 30||i==32||i==33||i== 34){ - continue; - } - // 每分钟小时 统计一次record 表 结果记录到统计表 - LocalDateTime now = LocalDateTime.now().minusMinutes(35-i); - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - int minus = DateUtil.nowMinus(now); - - Statistics s1 = new Statistics(); - s1.setKeyName("k21"); - s1.setApp("key21APP"); - s1.setCreateTime(nowTime); - s1.setDays(day); - s1.setHours(hour); - s1.setCount(rd.nextInt(800)); - s1.setBizType(1); - s1.setMinutes(minus); - s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s1); - - Statistics s11 = new Statistics(); - s11.setKeyName("k22"); - s11.setApp("key22APP"); - s11.setBizType(1); - s11.setCreateTime(nowTime); - s11.setDays(day); - s11.setMinutes(minus); - s11.setCount(rd.nextInt(600)); - s11.setHours(hour); - s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s11); - } - - if(1==1){ - return records; - } - // statisticsMapper.batchInsert(records); - List list = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - if(i == 12 || i ==13){ - }else{ - Statistics st = new Statistics(); - st.setApp("k21"); - st.setKeyName("k21"); - st.setCount(rd.nextInt(100)); - st.setBizType(1); - // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); - // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); - int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); - // System.out.println(time); - st.setMinutes(time); - // list.add(st); - } - } - List list2 = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - if(i == 0 || i == 1 || i == 18 || i == 19 || i == 25){ - - }else{ - Statistics st2 = new Statistics(); - st2.setApp("k22"); - st2.setKeyName("k22"); - st2.setCount(rd.nextInt(100)); - st2.setBizType(1); - st2.setMinutes(DateUtil.reviseTime(DateUtil.strToLdt("2006082355", DateUtil.PATTERN_MINUS),i,1)); - list2.add(st2); - } - } - // list.addAll(list2); - return list; - - } - - private List statisticsList1(int type){ - Random rd = new Random(); - List records = new ArrayList<>(); - statisticsMapper.clear(6); - - if(type == 3){ - int temp = 1; - for (int j = 0; j < 9; j++) { - for (int i = 0; i < 24; i++) { - // 每分钟小时 统计一次record 表 结果记录到统计表 - LocalDateTime now = LocalDateTime.now().minusHours(9 * 24 - temp); - - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - Statistics s1 = new Statistics(); - s1.setKeyName("k21"); - s1.setApp("key21APP"); - s1.setCreateTime(nowTime); - s1.setDays(day); - s1.setHours(hour); - s1.setCount(rd.nextInt(200)); - s1.setBizType(6); - s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s1); - - Statistics s11 = new Statistics(); - s11.setKeyName("k22"); - s11.setApp("key22APP"); - s11.setBizType(6); - s11.setCreateTime(nowTime); - s11.setDays(day); - s11.setCount(rd.nextInt(500)); - s11.setHours(hour); - s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s11); - - Statistics s33 = new Statistics(); - s33.setKeyName("k33"); - s33.setApp("key33APP"); - s33.setBizType(6); - s33.setCreateTime(nowTime); - s33.setDays(day); - s33.setCount(rd.nextInt(300)); - s33.setHours(hour); - s33.setUuid(3 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s33); - temp++; - } - } - - - }else{ - for (int i = 0; i < 30; i++) { - // 每分钟小时 统计一次record 表 结果记录到统计表 - LocalDateTime now = LocalDateTime.now().minusHours(30-i); - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - Statistics s1 = new Statistics(); - s1.setKeyName("key21"); - s1.setApp("key21APP"); - s1.setCreateTime(nowTime); - s1.setDays(day); - s1.setHours(hour); - s1.setCount(rd.nextInt(800)); - s1.setBizType(2); - s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s1); - - Statistics s11 = new Statistics(); - s11.setKeyName("key22"); - s11.setApp("key22APP"); - s11.setBizType(2); - s11.setCreateTime(nowTime); - s11.setDays(day); - s11.setCount(rd.nextInt(600)); - s11.setHours(hour); - s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); - records.add(s11); - } - } - statisticsMapper.batchInsert(records); - return records; - } - } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java index 5cb83e5..c03ea2e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java @@ -34,7 +34,7 @@ import java.util.Set; @Service public class UserServiceImpl implements UserService { - @Value("${erp.defaultPwd}") + @Value("${erp.defaultPwd:123}") private String defaultPwd; @Resource private UserMapper userMapper; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java new file mode 100644 index 0000000..cd4a987 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/PageUtil.java @@ -0,0 +1,36 @@ +package com.jd.platform.hotkey.dashboard.util; + +import com.google.common.collect.Lists; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-09-01 + */ +public class PageUtil { + /** + * 通用分页工具类 + * + * @param data + * @param pageSize + * @param pageNum + * @param + * @return + */ + public static Page pagination(final List data, final int pageSize, final int pageNum) { + if (CollectionUtils.isEmpty(data)) { + return new Page<>(1, 0, new ArrayList<>()); + } + List> lists = Lists.partition(data, pageSize); + int localPageNum = pageNum; + if (localPageNum >= lists.size()) { + localPageNum = lists.size() - 1; + } + return new Page<>(localPageNum, data.size(), lists.get(localPageNum)); + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java index 40d82ff..d9352dc 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/RuleUtil.java @@ -33,7 +33,7 @@ public class RuleUtil { } /** - * 根据APP的key，获取该key对应的rule + * 根据APP的key，获取该key对应的rule.如 cartpc-pu__ */ public static String rule(String key) { try { @@ -62,7 +62,7 @@ public class RuleUtil { return ""; } - private static KeyRule findByKey(String appNameKey) { + public static KeyRule findByKey(String appNameKey) { synchronized (RULE_MAP) { if (StrUtil.isEmpty(appNameKey)) { return null; diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml index f869987..94037b7 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml @@ -199,4 +199,10 @@ + + delete from hk_key_record + where app_name = #{app} + + + \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml deleted file mode 100644 index 04704cf..0000000 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml +++ /dev/null @@ -1,207 +0,0 @@ - - - - - - - - - - - - - - id, key_name, val, uuid, app_name, duration, create_time - - - - - delete from hk_key_timely - where key_name = #{key} and app_name = #{appName} - - - - - delete from hk_key_timely - where (app_name,key_name) in - - (#{item.appName},#{item.key}) - - - - - delete from hk_key_timely where 1 = 1 - - - - - INSERT ignore INTO - hk_key_timely( - key_name, - val, - app_name, - uuid, - duration, - create_time) - VALUES - - (#{k.key}, - #{k.val}, - #{k.appName}, - #{k.uuid}, - #{k.duration}, - #{k.createTime}) - - - - - insert into hk_key_timely - - - id, - - - key_name, - - - val, - - - uuid, - - - app_name, - - - duration, - - - create_time, - - - - - #{id,jdbcType=BIGINT}, - - - #{key,jdbcType=VARCHAR}, - - - #{val,jdbcType=VARCHAR}, - - - #{uuid,jdbcType=VARCHAR}, - - - #{appName,jdbcType=VARCHAR}, - - - #{duration,jdbcType=BIGINT}, - - - #{createTime,jdbcType=TIMESTAMP}, - - - - - - update hk_key_timely - - - key_name = #{key,jdbcType=VARCHAR}, - - - val = #{val,jdbcType=VARCHAR}, - - - uuid = #{uuid,jdbcType=VARCHAR}, - - - app_name = #{appName,jdbcType=VARCHAR}, - - - duration = #{duration,jdbcType=BIGINT}, - - - create_time = #{createTime,jdbcType=TIMESTAMP}, - - - where key_name = #{key} - - - - - - - - insert into hk_key_timely - - - key_name, - - - val, - - - uuid, - - - app_name, - - - duration, - - - create_time, - - - - - #{key,jdbcType=VARCHAR}, - - - #{val,jdbcType=VARCHAR}, - - - #{uuid,jdbcType=VARCHAR}, - - - #{appName,jdbcType=VARCHAR}, - - - #{duration,jdbcType=BIGINT}, - - - #{createTime,jdbcType=TIMESTAMP}, - - - on duplicate key update - create_time = now() - - - \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml index 8829d12..ac7f464 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml @@ -74,8 +74,9 @@ - + where app = #{app} + + \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml index 60f41a2..bd04545 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml @@ -51,4 +51,11 @@ + + delete from hk_summary + where app = #{app} + + + + \ No newline at end of file diff --git a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js index 05eb56e..01763f2 100644 --- a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js +++ b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js @@ -40,13 +40,6 @@ ajaxOptions:{ headers: {"Authorization": getCookie("token")} }, - exportOptions:{ - ignoreColumn: [0,1], //忽略某一列的索引 - fileName: '报表导出', //文件名称设置 - worksheetName: 'sheet1', //表格工作区名称 - tableName: '报表导出', - excelstyles: ['background-color', 'color', 'font-size', 'font-weight'] - }, onLoadSuccess: function(){ //加载成功时执行 var datas = $('#dataTable').bootstrapTable('getData'); console.info("加载成功"); @@ -70,7 +63,15 @@ if(app!==undefined && app != null && app!==""){ search.app = app; } - return search; + let time1 = $("#startTime").val(); + if(time1!==undefined && time1 != null && time1!==""){ + search.startTime = time1; + } + let time2 = $("#endTime").val(); + if(time2!==undefined && time2 != null && time2!==""){ + search.endTime = time2; + } + return search; }, search:function(my){//查询条件 diff --git a/dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js b/dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js new file mode 100644 index 0000000..0d8de56 --- /dev/null +++ b/dashboard/src/main/resources/static/admin/common/js/clearcfg-edit.js @@ -0,0 +1,25 @@ +$("#form-edit").validate({ + submitHandler:function(form){ + edit(); + } +}); + +function edit() { + let dataFormJson=$("#form-edit").serialize(); + $.ajax({ + cache : true, + type : "POST", + url : "/clear/save", + data : dataFormJson, + headers: { + "Authorization":getCookie("token") + }, + async : false, + error : function(XMLHttpRequest){ + $.modal.alertError(XMLHttpRequest.responseJSON.msg); + }, + success : function(data) { + $.operate.saveSuccess(data); + } + }); +} diff --git a/dashboard/src/main/resources/templates/admin/changeLog/list.html b/dashboard/src/main/resources/templates/admin/changeLog/list.html index a6fbf8d..d6ed97a 100644 --- a/dashboard/src/main/resources/templates/admin/changeLog/list.html +++ b/dashboard/src/main/resources/templates/admin/changeLog/list.html @@ -142,6 +142,11 @@ var options = { } } }, + { field: 'createTime', title: '创建时间', + formatter:function (val,row,index) { + return changeDateFormat(val); + } + }, { field: 'from', title: '初始值', diff --git a/dashboard/src/main/resources/templates/admin/clear/edit.html b/dashboard/src/main/resources/templates/admin/clear/edit.html new file mode 100644 index 0000000..4ced25c --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/clear/edit.html @@ -0,0 +1,69 @@ + + + + + + + + + + + + + + + + + +
+
+
+
+
+
+
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+ +
+
+ + +
+
+
+
+
+
+
+
+
+ + +
+ +
+ + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/clear/list.html b/dashboard/src/main/resources/templates/admin/clear/list.html new file mode 100644 index 0000000..7087f79 --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/clear/list.html @@ -0,0 +1,157 @@ + + + + + + + + + + + + + + + +
+
+ +
+ +
+
+ +
+
+
搜索条件
+
+ +
+
+
+
+ + +
+ +
+
+
+
+ + +
+
+
表单
+
+ + + +
+
+ + + +
+
+ +
+ +
+
+
+
+ +
+ +
+
+
+ + + +
+ +
+ + + + + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index 29baffe..668341d 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -52,7 +52,7 @@ - + 热点记录 变更记录 + + + 数据清理配置 + diff --git a/dashboard/src/main/resources/templates/admin/key/list.html b/dashboard/src/main/resources/templates/admin/key/list.html index 38c596f..6f755c8 100644 --- a/dashboard/src/main/resources/templates/admin/key/list.html +++ b/dashboard/src/main/resources/templates/admin/key/list.html @@ -163,7 +163,7 @@ var options = { { field: 'key', title: 'key'}, { field: 'ruleDesc', title: '类型'}, { field: 'appName', title: '所属APP'}, - { field: 'duration', title: '缓存时间'}, + { field: 'duration', title: '剩余缓存时间'}, { field: 'source', title: '来源'}, { field: 'type', title: '操作类型', formatter:function (val) { @@ -174,8 +174,27 @@ var options = { { field: 'createTime', title: '创建时间' }] }; -$(function(){ + + $('.datetimepicker').datetimepicker({ + language: 'zh-CN', + CustomFormat: 'yyyy-mm-dd HH:ii:ss', + weekStart: 1, + todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 + autoclose: 1, //选完时间自动关闭 + todayHighlight: 1, //当天时间高亮 + startView: 2, //从月视图开始，选天 + minView: 0, //提供选择分钟的视图 + forceParse: 0, + minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 + }); + let now = new Date(); + $("#datetimepicker2").datetimepicker("setDate", now); + let pre5hours = now.setHours(now.getHours() - 5); + $("#datetimepicker1").datetimepicker("setDate", new Date(pre5hours)); + + +$(function(){ $.ajax({ cache : true, type : "POST", @@ -206,24 +225,9 @@ $(function(){ } }); - var oTab=$.table.oTableInit(options); oTab.Init(); - $('.datetimepicker').datetimepicker({ - language: 'zh-CN', - CustomFormat: 'yyyy-mm-dd HH:ii:ss', - weekStart: 1, - todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 - autoclose: 1, //选完时间自动关闭 - todayHighlight: 1, //当天时间高亮 - startView: 2, //从月视图开始，选天 - minView: 0, //提供选择分钟的视图 - forceParse: 0, - minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 -}); - - }) function exportHot() { diff --git a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html index 366b156..5d7fc22 100644 --- a/dashboard/src/main/resources/templates/admin/key/listmaxhot.html +++ b/dashboard/src/main/resources/templates/admin/key/listmaxhot.html @@ -35,14 +35,14 @@
- + - + diff --git a/dashboard/src/main/resources/templates/admin/key/listtimely.html b/dashboard/src/main/resources/templates/admin/key/listtimely.html index 37e85be..545caa7 100644 --- a/dashboard/src/main/resources/templates/admin/key/listtimely.html +++ b/dashboard/src/main/resources/templates/admin/key/listtimely.html @@ -162,7 +162,7 @@ var options = { { field: 'key', title: 'key'}, { field: 'ruleDesc', title: '类型'}, { field: 'appName', title: '所属APP'}, - { field: 'duration', title: '缓存时间'}, + { field: 'duration', title: '剩余缓存时间'}, { field: 'createTime', title: '创建时间', formatter:function (val,row,index) { return changeDateFormat(val); diff --git a/dashboard/src/main/resources/templates/admin/rule/listhitcount.html b/dashboard/src/main/resources/templates/admin/rule/listhitcount.html index 7e501fa..f1be6e3 100644 --- a/dashboard/src/main/resources/templates/admin/rule/listhitcount.html +++ b/dashboard/src/main/resources/templates/admin/rule/listhitcount.html @@ -46,7 +46,7 @@
- + @@ -55,7 +55,7 @@
- + @@ -134,6 +134,27 @@ } }] }; + + + + $('.datetimepicker').datetimepicker({ + language: 'zh-CN', + CustomFormat: 'yyyy-mm-dd HH:ii:ss', + weekStart: 1, + todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 + autoclose: 1, //选完时间自动关闭 + todayHighlight: 1, //当天时间高亮 + startView: 2, //从月视图开始，选天 + minView: 0, //提供选择分钟的视图 + forceParse: 0, + minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 + }); + let now = new Date(); + $("#datetimepicker2").datetimepicker("setDate", now); + let pre5hours = now.setHours(now.getHours() - 5); + $("#datetimepicker1").datetimepicker("setDate", new Date(pre5hours)); + + $(function(){ $.ajax({ cache : true, @@ -164,26 +185,12 @@ } } }); - var oTab=$.table.oTableInit(options); + + let oTab=$.table.oTableInit(options); oTab.Init(); }) - - $('.datetimepicker').datetimepicker({ - language: 'zh-CN', - CustomFormat: 'yyyy-mm-dd HH:ii:ss', - weekStart: 1, - todayBtn: 1, //显示当天按钮，点击则选择当天当天时间 - autoclose: 1, //选完时间自动关闭 - todayHighlight: 1, //当天时间高亮 - startView: 2, //从月视图开始，选天 - minView: 0, //提供选择分钟的视图 - forceParse: 0, - minuteStep: 1 //用于构建小时视图。就是最小的视图是每1分钟可选一次。是以分钟为单位的 - }); - - diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java deleted file mode 100644 index 0d62c90..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/config/CodecConfig.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.jd.platform.hotkey.worker.config; - -import com.jd.platform.hotkey.common.coder.Codec; -import com.jd.platform.hotkey.common.coder.NettyCodec; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * netty的编解码器 - * @author wuweifeng wrote on 2020-02-21 - * @version 1.0 - */ -@Configuration -public class CodecConfig { - - @Bean - public Codec codec() { - return new NettyCodec(); - } -} -- Gitee From 362d33d35c21d20b0f47a2b37b2af01c37332962 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 16:09:04 +0800 Subject: [PATCH 16/59] =?UTF-8?q?worker=E5=90=91dashboard=E7=9B=B4?= =?UTF-8?q?=E6=8E=A8=E7=83=ADkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/common/tool/flush/FlushUtil.java | 18 ++++ .../worker/disruptor/AbsEventConsumer.java | 54 ------------ .../worker/disruptor/AbsWorkConsumer.java | 54 ------------ .../hotkey/worker/disruptor/BaseEvent.java | 26 ------ .../worker/disruptor/DisruptorBuilder.java | 65 -------------- .../worker/disruptor/MessageProducer.java | 8 -- .../worker/disruptor/ProducerFactory.java | 50 ----------- .../worker/disruptor/hotkey/HotKeyEvent.java | 18 ---- .../disruptor/hotkey/HotKeyEventConsumer.java | 41 --------- .../disruptor/hotkey/HotKeyEventProducer.java | 32 ------- .../worker/keydispatcher/KeyProducer.java | 5 +- .../hotkey/worker/mq/IMqMessageReceiver.java | 9 -- .../netty/dashboard/DashboardHolder.java | 26 ++++++ .../worker/netty/dashboard/NettyClient.java | 86 +++++++++++++++++++ .../netty/dashboard/NettyClientHandler.java | 58 +++++++++++++ .../worker/netty/filter/HeartBeatFilter.java | 2 +- .../worker/netty/filter/HotKeyFilter.java | 13 +-- .../worker/netty/filter/KeyCounterFilter.java | 7 +- .../hotkey/worker/netty/flush/FlushUtil.java | 35 -------- .../netty/pusher/BatchToEtcdScheduler.java | 45 ---------- .../worker/netty/pusher/DashboardPusher.java | 24 ++++++ .../worker/netty/pusher/EtcdPusher.java | 49 ----------- .../worker/netty/pusher/KeyCollector.java | 53 ------------ .../worker/netty/pusher/KeyUploader.java | 46 ++++++++++ .../netty/pusher/store/HotkeyTempStore.java | 27 ++++++ .../worker/netty/server/NodesServer.java | 5 ++ .../hotkey/worker/starters/EtcdStarter.java | 25 ++++++ 27 files changed, 323 insertions(+), 558 deletions(-) delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java delete mode 100755 worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java index 83e74c1..af411d0 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.common.tool.flush; +import com.jd.platform.hotkey.common.model.HotKeyMsg; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; @@ -29,6 +30,23 @@ public class FlushUtil { } catch (InterruptedException e) { logger.error("flush error " + e.getMessage()); } +// } + } + + public static void flush(ChannelHandlerContext channelHandlerContext, HotKeyMsg hotKeyMsg) { +// if (channelHandlerContext.channel().isWritable()) { +// channelHandlerContext.channel().writeAndFlush(hotKeyMsg).addListener(future -> { +// if (!future.isSuccess()) { +// logger.warn("flush error " + future.cause().getMessage()); +// } +// }); +// } else { + try { + //同步发送 + channelHandlerContext.channel().writeAndFlush(hotKeyMsg).sync(); + } catch (InterruptedException e) { + logger.error("flush error " + e.getMessage()); + } // } } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java deleted file mode 100755 index 47839ae..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsEventConsumer.java +++ /dev/null @@ -1,54 +0,0 @@ -//package com.jd.platform.hotkey.worker.disruptor; -// -//import cn.hutool.core.date.SystemClock; -//import com.jd.platform.hotkey.common.model.BaseModel; -//import com.jd.platform.hotkey.common.tool.Constant; -//import com.jd.platform.hotkey.worker.tool.InitConstant; -//import com.lmax.disruptor.EventHandler; -// -//import java.util.concurrent.atomic.LongAdder; -//import java.util.zip.CRC32; -// -///**各个消费者重复消费 -// * @author wuweifeng wrote on 2019-08-21. -// */ -//public abstract class AbsEventConsumer implements EventHandler { -// -// private int hashIndex; -// -// public static final LongAdder totalDealCount = new LongAdder(); -// //过期的 -// public static final LongAdder expireTotalCount = new LongAdder(); -// -// public AbsEventConsumer(int hashIndex) { -// this.hashIndex = hashIndex; -// } -// -// @Override -// public void onEvent(T t, long l, boolean b) { -// //每个消费者，只处理特定的key。保证相同的key，一定被同一个线程处理 -// BaseModel model = t.getModel(); -// if (model == null || model.getKey() == null) { -// return; -// } -// if (Math.abs(hash(model.getKey())) % Constant.Default_Threads == hashIndex) { -// //5秒前的过时消息就不处理了 -// if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { -// expireTotalCount.increment(); -// return; -// } -// onNewEvent(t); -// -// //处理完毕，将数量加1 -// totalDealCount.increment(); -// } -// } -// -// protected abstract void onNewEvent(T t); -// -// private long hash(String key) { -// CRC32 crc = new CRC32(); -// crc.update(key.getBytes()); -// return crc.getValue(); -// } -//} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java deleted file mode 100755 index bca2e70..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/AbsWorkConsumer.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import cn.hutool.core.date.SystemClock; -import com.jd.platform.hotkey.common.model.BaseModel; -import com.jd.platform.hotkey.worker.tool.InitConstant; -import com.lmax.disruptor.WorkHandler; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.concurrent.atomic.LongAdder; - -/** - * 各个消费者不重复消费 - * - * @author wuweifeng wrote on 2019-08-21. - */ -public abstract class AbsWorkConsumer implements WorkHandler { - - private int hashIndex; - - public static final LongAdder totalDealCount = new LongAdder(); - //过期的 - public static final LongAdder expireTotalCount = new LongAdder(); - - private Logger logger = LoggerFactory.getLogger(getClass()); - - public AbsWorkConsumer(int hashIndex) { - this.hashIndex = hashIndex; - } - - @Override - public void onEvent(T t) { - BaseModel model = t.getModel(); - if (model == null || model.getKey() == null) { - return; - } - //5秒前的过时消息就不处理了 - if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { - expireTotalCount.increment(); - logger.warn("this key timeout : " + model.getKey() + " createAt : " + model.getCreateTime() + - " event at " + SystemClock.now()); - if (InitConstant.openTimeOut) { - return; - } - } - onNewEvent(t); - - //处理完毕，将数量加1 - totalDealCount.increment(); - } - - protected abstract void onNewEvent(T t); - -} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java deleted file mode 100644 index e8483fe..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/BaseEvent.java +++ /dev/null @@ -1,26 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import com.jd.platform.hotkey.common.model.BaseModel; - -/** - * @author wuweifeng wrote on 2019-12-10 - * @version 1.0 - */ -public class BaseEvent { - private T model; - - public BaseEvent(T model) { - this.model = model; - } - - public BaseEvent() { - } - - public T getModel() { - return model; - } - - public void setModel(T model) { - this.model = model; - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java deleted file mode 100644 index 64dc4c0..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/DisruptorBuilder.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import com.lmax.disruptor.BlockingWaitStrategy; -import com.lmax.disruptor.EventFactory; -import com.lmax.disruptor.EventHandler; -import com.lmax.disruptor.WorkHandler; -import com.lmax.disruptor.dsl.Disruptor; -import com.lmax.disruptor.dsl.ProducerType; - -import java.util.concurrent.Executors; - -/** - * 每个worker重复消费生产者消息 - * - * @author wuweifeng wrote on 2019-12-10 - * @version 1.0 - */ -public class DisruptorBuilder { - private int bufferSize; - private EventHandler[] eventHandlers; - private WorkHandler[] workHandlers; - private EventFactory eventFactory; - - public Disruptor build() { - Disruptor disruptor = new Disruptor<>(eventFactory, bufferSize, Executors.defaultThreadFactory(), - ProducerType.SINGLE, new BlockingWaitStrategy()); - - if (eventHandlers != null) { - disruptor.handleEventsWith(eventHandlers); - } - if (workHandlers != null) { - disruptor.handleEventsWithWorkerPool(workHandlers); - } - - disruptor.start(); - return disruptor; - } - - /** - * 每个worker会重复消费 - */ - public DisruptorBuilder setEventHandlers(EventHandler... eventHandlers) { - this.eventHandlers = eventHandlers; - return this; - } - - /** - * 每个worker不会重复消费 - */ - public DisruptorBuilder setWorkerHandlers(WorkHandler... workHandlers) { - this.workHandlers = workHandlers; - return this; - } - - public DisruptorBuilder setEventFactory(EventFactory eventFactory) { - this.eventFactory = eventFactory; - return this; - } - - public DisruptorBuilder setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - return this; - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java deleted file mode 100755 index 63d1421..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/MessageProducer.java +++ /dev/null @@ -1,8 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -/** - * @author wuweifeng wrote on 2018/4/20. - */ -public interface MessageProducer { - void publish(T t); -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java deleted file mode 100755 index 8854627..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/ProducerFactory.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor; - -import com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEvent; -import com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEventConsumer; -import com.jd.platform.hotkey.worker.disruptor.hotkey.HotKeyEventProducer; -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import com.jd.platform.hotkey.worker.tool.CpuNum; -import com.jd.platform.hotkey.worker.tool.InitConstant; -import com.lmax.disruptor.dsl.Disruptor; - -/** - * Disruptor创建producer的地方 - * - * @author wuweifeng wrote on 2019-11-5. - */ -public class ProducerFactory { - - /** - * 创建热key的producer - * - * @return HotKeyEventProducer - */ - public static MessageProducer createHotKeyProducer(IKeyListener iKeyListener) { - int threadCount = CpuNum.workerCount(); - //如果手工指定了线程数，就用手工指定的 - if (InitConstant.threadCount != 0) { - threadCount = InitConstant.threadCount; - } else { - if (threadCount >= 8) { - threadCount = threadCount / 2; - } - } - - HotKeyEventConsumer[] array = new HotKeyEventConsumer[threadCount]; - for (int i = 0; i < threadCount; i++) { - array[i] = new HotKeyEventConsumer(i); - array[i].setKeyListener(iKeyListener); - } - DisruptorBuilder builder = new DisruptorBuilder<>(); - Disruptor disruptor = builder - .setBufferSize(InitConstant.bufferSize * 1024 * 1024) - .setEventFactory(HotKeyEvent::new) -// .setEventHandlers(array) //重复消费的 - .setWorkerHandlers(array) //不重复消费的 - .build(); - - return new HotKeyEventProducer(disruptor); - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java deleted file mode 100755 index 8ec41c3..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEvent.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor.hotkey; - -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.worker.disruptor.BaseEvent; - -/** - * @author wuweifeng wrote on 2019-08-21. - */ -public class HotKeyEvent extends BaseEvent { - - public HotKeyEvent(HotKeyModel hotKeyModel) { - super(hotKeyModel); - } - - public HotKeyEvent() { - - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java deleted file mode 100755 index ca248f5..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventConsumer.java +++ /dev/null @@ -1,41 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor.hotkey; - -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.worker.disruptor.AbsWorkConsumer; -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import com.jd.platform.hotkey.worker.keylistener.KeyEventOriginal; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * 热key的消费者 - * @author wuweifeng wrote on 2019-12-12. - */ -public class HotKeyEventConsumer extends AbsWorkConsumer { - private Logger logger = LoggerFactory.getLogger(getClass()); - - private IKeyListener iKeyListener; - - public HotKeyEventConsumer(int hashIndex) { - super(hashIndex); - } - - @Override - protected void onNewEvent(HotKeyEvent hotKeyEvent) { - HotKeyModel model = hotKeyEvent.getModel(); - if (iKeyListener == null) { - logger.warn("new key is coming, but no consumer deal this key!"); - return; - } - - if (model.isRemove()) { - iKeyListener.removeKey(model, KeyEventOriginal.CLIENT); - } else { - iKeyListener.newKey(model, KeyEventOriginal.CLIENT); - } - } - - public void setKeyListener(IKeyListener iKeyListener) { - this.iKeyListener = iKeyListener; - } -} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java deleted file mode 100755 index 42dae5f..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/disruptor/hotkey/HotKeyEventProducer.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.jd.platform.hotkey.worker.disruptor.hotkey; - -import com.jd.platform.hotkey.worker.disruptor.MessageProducer; -import com.lmax.disruptor.RingBuffer; -import com.lmax.disruptor.dsl.Disruptor; - -/** - * 所有节点发来的消息，都进入这里，然后publish出去，供消费者消费 - * - * @author wuweifeng wrote on 2019/11/6. - */ -public class HotKeyEventProducer implements MessageProducer { - - private Disruptor disruptor; - - public HotKeyEventProducer(Disruptor disruptor) { - this.disruptor = disruptor; - } - - @Override - public void publish(HotKeyEvent hotKeyEvent) { - RingBuffer ringBuffer = disruptor.getRingBuffer(); - long sequence = ringBuffer.next(); - try { - HotKeyEvent event = ringBuffer.get(sequence); - event.setModel(hotKeyEvent.getModel()); - } finally { - ringBuffer.publish(sequence); - } - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java index fc86ecc..2f3320b 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keydispatcher/KeyProducer.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.worker.keydispatcher; -import cn.hutool.core.date.SystemClock; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.worker.tool.InitConstant; import org.springframework.stereotype.Component; @@ -17,12 +16,12 @@ import static com.jd.platform.hotkey.worker.tool.InitConstant.totalOfferCount; @Component public class KeyProducer { - public void push(HotKeyModel model) { + public void push(HotKeyModel model, long now) { if (model == null || model.getKey() == null) { return; } //5秒前的过时消息就不处理了 - if (SystemClock.now() - model.getCreateTime() > InitConstant.timeOut) { + if (now - model.getCreateTime() > InitConstant.timeOut) { expireTotalCount.increment(); return; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java b/worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java deleted file mode 100644 index 155a476..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/mq/IMqMessageReceiver.java +++ /dev/null @@ -1,9 +0,0 @@ -package com.jd.platform.hotkey.worker.mq; - -/** - * @author wuweifeng wrote on 2019-12-12 - * @version 1.0 - */ -public interface IMqMessageReceiver { - void receive(String msg); -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java new file mode 100644 index 0000000..6e0d57e --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java @@ -0,0 +1,26 @@ +package com.jd.platform.hotkey.worker.netty.dashboard; + +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.MsgBuilder; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import io.netty.channel.Channel; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +public class DashboardHolder { + /** + * 是否连上了dashboard + */ + public static boolean hasConnected = false; + /** + * channel + */ + public static Channel channel = null; + + public static void flushToDashboard(String message) { + channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_HOT_KEY, message))); + } +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java new file mode 100644 index 0000000..5209c35 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClient.java @@ -0,0 +1,86 @@ +package com.jd.platform.hotkey.worker.netty.dashboard; + +import com.jd.platform.hotkey.common.tool.Constant; +import io.netty.bootstrap.Bootstrap; +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; +import io.netty.channel.*; +import io.netty.channel.nio.NioEventLoopGroup; +import io.netty.channel.socket.SocketChannel; +import io.netty.channel.socket.nio.NioSocketChannel; +import io.netty.handler.codec.DelimiterBasedFrameDecoder; +import io.netty.handler.codec.string.StringDecoder; +import io.netty.handler.timeout.IdleStateHandler; + +/** + * netty连接器 + * + * @author wuweifeng wrote on 2019-11-05. + */ +public class NettyClient { + private static final NettyClient nettyClient = new NettyClient(); + + private Bootstrap bootstrap; + + + public static NettyClient getInstance() { + return nettyClient; + } + + private NettyClient() { + if (bootstrap == null) { + bootstrap = initBootstrap(); + } + } + + private Bootstrap initBootstrap() { + //少线程 + EventLoopGroup group = new NioEventLoopGroup(2); + + Bootstrap bootstrap = new Bootstrap(); + NettyClientHandler nettyClientHandler = new NettyClientHandler(); + bootstrap.group(group).channel(NioSocketChannel.class) + .option(ChannelOption.SO_KEEPALIVE, true) + .option(ChannelOption.TCP_NODELAY, true) + .handler(new ChannelInitializer() { + @Override + protected void initChannel(SocketChannel ch) { + ByteBuf delimiter = Unpooled.copiedBuffer(Constant.DELIMITER.getBytes()); + ch.pipeline() + .addLast(new DelimiterBasedFrameDecoder(Constant.MAX_LENGTH, delimiter)) + .addLast(new StringDecoder()) + //10秒没消息时，就发心跳包过去 + .addLast(new IdleStateHandler(0, 0, 30)) + .addLast(nettyClientHandler); + } + }); + return bootstrap; + } + + public synchronized void connect(String address) { + if (DashboardHolder.hasConnected) { + return; + } + String[] ss = address.split(":"); + try { + ChannelFuture channelFuture = bootstrap.connect(ss[0], Integer.parseInt(ss[1])).sync(); + DashboardHolder.channel = channelFuture.channel(); + DashboardHolder.hasConnected = true; + } catch (Exception e) { + DashboardHolder.hasConnected = false; + DashboardHolder.channel = null; + e.printStackTrace(); + } + + //这一步就阻塞了 +// channelFuture.channel().closeFuture().sync(); + //当server断开后才会走下面的 +// System.out.println("server is down"); + } + + public synchronized void disConnect() { + DashboardHolder.channel = null; + DashboardHolder.hasConnected = false; + } + +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java new file mode 100644 index 0000000..5fdbe3a --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/NettyClientHandler.java @@ -0,0 +1,58 @@ +package com.jd.platform.hotkey.worker.netty.dashboard; + +import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.MsgBuilder; +import com.jd.platform.hotkey.common.model.typeenum.MessageType; +import com.jd.platform.hotkey.common.tool.Constant; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import io.netty.channel.ChannelHandler; +import io.netty.channel.ChannelHandlerContext; +import io.netty.channel.SimpleChannelInboundHandler; +import io.netty.handler.timeout.IdleState; +import io.netty.handler.timeout.IdleStateEvent; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * @author wuweifeng wrote on 2019-11-05. + */ +@ChannelHandler.Sharable +public class NettyClientHandler extends SimpleChannelInboundHandler { + + private Logger logger = LoggerFactory.getLogger(getClass()); + + @Override + public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { + if (evt instanceof IdleStateEvent) { + IdleStateEvent idleStateEvent = (IdleStateEvent) evt; + + if (idleStateEvent.state() == IdleState.ALL_IDLE) { + //向服务端发送消息 + ctx.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.PING, Constant.PING))); + } + } + + super.userEventTriggered(ctx, evt); + } + + @Override + public void channelActive(ChannelHandlerContext ctx) { + logger.info("channelActive:" + ctx.name()); + } + + @Override + public void channelInactive(ChannelHandlerContext ctx) throws Exception { + super.channelInactive(ctx); + NettyClient.getInstance().disConnect(); + } + + @Override + protected void channelRead0(ChannelHandlerContext channelHandlerContext, String message) { + HotKeyMsg msg = FastJsonUtils.toBean(message, HotKeyMsg.class); + if (MessageType.PONG == msg.getMessageType()) { + logger.info("heart beat"); + } + + } + +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java index 82112ca..bc8171e 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java @@ -2,7 +2,7 @@ package com.jd.platform.hotkey.worker.netty.filter; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.worker.netty.flush.FlushUtil; +import com.jd.platform.hotkey.common.tool.flush.FlushUtil; import io.netty.channel.ChannelHandlerContext; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index 2e37f08..e362c18 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -7,7 +7,6 @@ import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.keydispatcher.KeyProducer; -import com.jd.platform.hotkey.worker.mq.IMqMessageReceiver; import com.jd.platform.hotkey.worker.netty.holder.WhiteListHolder; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; @@ -27,7 +26,7 @@ import java.util.concurrent.atomic.AtomicLong; */ @Component @Order(3) -public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { +public class HotKeyFilter implements INettyMsgFilter { @Resource private KeyProducer keyProducer; @@ -48,14 +47,10 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { return true; } - @Override - public void receive(String msg) { -// publishMsg(msg, null); - } - private void publishMsg(HotKeyMsg message, ChannelHandlerContext ctx) { //老版的用的单个HotKeyModel，新版用的数组 List models = message.getHotKeyModels(); + long now = SystemClock.now(); if (CollectionUtil.isEmpty(models)) { return; } @@ -64,11 +59,11 @@ public class HotKeyFilter implements INettyMsgFilter, IMqMessageReceiver { if (WhiteListHolder.contains(model.getKey())) { continue; } - long timeOut = SystemClock.now() - model.getCreateTime(); + long timeOut = now - model.getCreateTime(); if (timeOut > 1000) { logger.info("key timeout " + timeOut + ", from ip : " + NettyIpUtil.clientIp(ctx)); } - keyProducer.push(model); + keyProducer.push(model, now); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java index 0022157..115200f 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java @@ -8,7 +8,6 @@ import com.jd.platform.hotkey.common.model.KeyCountModel; import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.counter.KeyCountItem; -import com.jd.platform.hotkey.worker.mq.IMqMessageReceiver; import com.jd.platform.hotkey.worker.tool.InitConstant; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; @@ -28,7 +27,7 @@ import static com.jd.platform.hotkey.worker.counter.CounterConfig.COUNTER_QUEUE; */ @Component @Order(4) -public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { +public class KeyCounterFilter implements INettyMsgFilter { private Logger logger = LoggerFactory.getLogger(getClass()); /** @@ -52,10 +51,6 @@ public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { return true; } - @Override - public void receive(String msg) { -// publishMsg("", msg, null); - } private void publishMsg(String appName, HotKeyMsg message, ChannelHandlerContext ctx) { //老版的用的单个HotKeyModel，新版用的数组 diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java deleted file mode 100644 index c9f3ed0..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/flush/FlushUtil.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.flush; - -import com.jd.platform.hotkey.common.model.HotKeyMsg; -import io.netty.channel.ChannelHandlerContext; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -public class FlushUtil { - private static Logger logger = LoggerFactory.getLogger("flushUtil"); - - /** - * 往channel里输出消息 - */ - - public static void flush(ChannelHandlerContext channelHandlerContext, HotKeyMsg hotKeyMsg) { -// if (channelHandlerContext.channel().isWritable()) { -// channelHandlerContext.channel().writeAndFlush(hotKeyMsg).addListener(future -> { -// if (!future.isSuccess()) { -// logger.warn("flush error " + future.cause().getMessage()); -// } -// }); -// } else { - try { - //同步发送 - channelHandlerContext.channel().writeAndFlush(hotKeyMsg).sync(); - } catch (InterruptedException e) { - logger.error("flush error " + e.getMessage()); - } -// } - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java deleted file mode 100644 index fe958d5..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/BatchToEtcdScheduler.java +++ /dev/null @@ -1,45 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher; - -import com.jd.platform.hotkey.common.configcenter.ConfigConstant; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.util.Set; -import java.util.UUID; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-05-26 - */ -@Component -@Deprecated -public class BatchToEtcdScheduler { - @Resource - private KeyCollector keyCollector; - @Resource - private IConfigCenter iConfigCenter; - - /** - * 每隔0.5秒上传一下已探测出的热key发往etcd供入库 - */ -// @Scheduled(fixedRate = 500) - public void uploadClientCount() { - Set set = keyCollector.lockAndGetResult(); - if (set.size() == 0) { - return; - } - - //worker将热key推送到该地址，供dashboard监听入库做记录 - String hotKeyRecordPath = ConfigConstant.hotKeyRecordPath + UUID.randomUUID().toString(); - try { - //推送到etcd，供dashboard监听入库 - iConfigCenter.putAndGrant(hotKeyRecordPath, FastJsonUtils.convertObjectToJSON(set), 10); - } catch (Exception e) { - e.printStackTrace(); - } - - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java new file mode 100644 index 0000000..aa420cd --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java @@ -0,0 +1,24 @@ +package com.jd.platform.hotkey.worker.netty.pusher; + +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.worker.netty.pusher.store.HotkeyTempStore; +import org.springframework.stereotype.Component; + +/** + * 将热key推送到dashboard供入库 + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +@Component +public class DashboardPusher implements IPusher { + @Override + public void push(HotKeyModel model) { + HotkeyTempStore.push(model); + } + + @Override + public void remove(HotKeyModel model) { + + } +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java deleted file mode 100644 index aa1971b..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/EtcdPusher.java +++ /dev/null @@ -1,49 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher; - -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.common.tool.HotKeyPathTool; -import com.jd.platform.hotkey.worker.rule.KeyRuleHolder; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.util.UUID; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - - -/** - * 将热key推送到etcd，dashboard监听后留做备份 - * @author wuweifeng wrote on 2020-02-24 - * @version 1.0 - */ -@Component -public class EtcdPusher implements IPusher { - @Resource - private IConfigCenter iConfigCenter; - - /** - * 将推送key到etcd的任务都丢到这个线程池里 - */ - private static final ExecutorService EXECUTOR_SERVICE = Executors.newFixedThreadPool(128); - - @Override - public void push(HotKeyModel model) { - //2020-8-25日，新增线程池写入etcd的逻辑。本地测试发现8线程写入etcd，每秒最多500个key-value，128线程能写2500个以上 - EXECUTOR_SERVICE.submit(() -> { - //推送到etcd，供dashboard监听入库。 - iConfigCenter.putAndGrant(HotKeyPathTool.keyRecordPath(model), UUID.randomUUID().toString(), - KeyRuleHolder.getRuleByAppAndKey(model).getDuration()); - }); - - } - - @Override - @Deprecated - public void remove(HotKeyModel model) { - //推送etcd删除 - iConfigCenter.delete(HotKeyPathTool.keyPath(model)); - } - - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java deleted file mode 100644 index 1dad842..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyCollector.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher; - -import cn.hutool.core.collection.ConcurrentHashSet; -import cn.hutool.core.util.StrUtil; -import com.jd.platform.hotkey.common.model.HotKeyModel; -import org.springframework.stereotype.Component; - -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; - -import static com.jd.platform.hotkey.common.tool.KeyRecordKeyTool.key; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-05-26 - */ -@Component -@Deprecated -public class KeyCollector { - private Set keySet0 = new ConcurrentHashSet<>(); - private Set keySet1 = new ConcurrentHashSet<>(); - private AtomicLong atomicLong = new AtomicLong(0); - - public Set lockAndGetResult() { - //自增后，对应的map就会停止被写入，等待被读取 - atomicLong.addAndGet(1); - Set set; - if (atomicLong.get() % 2 == 0) { - set = new HashSet<>(keySet1); - keySet1.clear(); - } else { - set = new HashSet<>(keySet0); - keySet0.clear(); - } - return set; - } - - public void collect(HotKeyModel hotKeyModel) { - String key = hotKeyModel.getKey(); - if (StrUtil.isEmpty(key)) { - return; - } - if (atomicLong.get() % 2 == 0) { - keySet0.add(key(hotKeyModel)); - } else { - keySet1.add(key(hotKeyModel)); - } - - } - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java new file mode 100644 index 0000000..8ee6e11 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java @@ -0,0 +1,46 @@ +package com.jd.platform.hotkey.worker.netty.pusher; + +import cn.hutool.core.collection.CollectionUtil; +import com.google.common.collect.Queues; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.worker.netty.dashboard.DashboardHolder; +import com.jd.platform.hotkey.worker.netty.pusher.store.HotkeyTempStore; +import com.jd.platform.hotkey.worker.tool.AsyncPool; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** + * 定时批量往dashboard发送热key，供入库 + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +@Component +public class KeyUploader { + + @PostConstruct + public void uploadToDashboard() { + AsyncPool.asyncDo(() -> { + while (true) { + try { + //要么key达到1千个，要么达到1秒，就汇总上报给etcd一次 + List tempModels = new ArrayList<>(); + Queues.drain(HotkeyTempStore.getQueue(), tempModels, 1000, 1, TimeUnit.SECONDS); + if (CollectionUtil.isEmpty(tempModels)) { + continue; + } + + //将热key推到dashboard + DashboardHolder.flushToDashboard(FastJsonUtils.convertObjectToJSON(tempModels)); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java new file mode 100644 index 0000000..c93d18d --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java @@ -0,0 +1,27 @@ +package com.jd.platform.hotkey.worker.netty.pusher.store; + +import com.jd.platform.hotkey.common.model.HotKeyModel; + +import java.util.concurrent.LinkedBlockingQueue; + +/** + * 已热待上报入库的热key集中营 + * @author wuweifeng + * @version 1.0 + * @date 2020-08-31 + */ +public class HotkeyTempStore { + /** + * 热key集中营 + */ + private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); + + public static void push(HotKeyModel model) { + hotKeyStoreQueue.offer(model); + } + + public static LinkedBlockingQueue getQueue() { + return hotKeyStoreQueue; + } + +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java index 606e07f..93f80dd 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServer.java @@ -17,6 +17,7 @@ import io.netty.handler.logging.LogLevel; import io.netty.handler.logging.LoggingHandler; import java.util.List; +import java.util.concurrent.TimeUnit; /** * 该server用于给各个微服务实例连接用。 @@ -43,6 +44,10 @@ public class NodesServer { .childHandler(new ChildChannelHandler()); //绑定端口，同步等待成功 ChannelFuture future = bootstrap.bind(port).sync(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + bossGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + workerGroup.shutdownGracefully (1000, 3000, TimeUnit.MILLISECONDS); + })); //等待服务器监听端口关闭 future.channel().closeFuture().sync(); } catch (Exception e) { diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java index 9fe62a4..2c60d6e 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.worker.starters; +import cn.hutool.core.collection.CollectionUtil; import cn.hutool.core.util.StrUtil; import com.ibm.etcd.api.Event; import com.ibm.etcd.api.KeyValue; @@ -13,6 +14,7 @@ import com.jd.platform.hotkey.common.tool.IpUtils; import com.jd.platform.hotkey.worker.cache.CaffeineCacheHolder; import com.jd.platform.hotkey.worker.model.AppInfo; import com.jd.platform.hotkey.worker.model.TotalCount; +import com.jd.platform.hotkey.worker.netty.dashboard.NettyClient; import com.jd.platform.hotkey.worker.netty.filter.HotKeyFilter; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; import com.jd.platform.hotkey.worker.netty.holder.WhiteListHolder; @@ -217,6 +219,29 @@ public class EtcdStarter { } } + /** + * 每隔30秒去获取一下dashboard的地址 + */ + @Scheduled(fixedRate = 30000) + public void fetchDashboardIp() { + try { + //获取DashboardIp + List keyValues = configCenter.getPrefix(ConfigConstant.dashboardPath); + + //是空，给个警告 + if (CollectionUtil.isEmpty(keyValues)) { + logger.warn("very important warn !!! Dashboard ip is null!!!"); + return; + } + + String dashboardIp = keyValues.get(0).getValue().toStringUtf8(); + NettyClient.getInstance().connect(dashboardIp); + + } catch (Exception e) { + e.printStackTrace(); + } + } + /** * 校验一下receive的key数量，如果一段时间没变，考虑网络问题，就将worker注册自己到etcd的心跳给断掉30秒，让各client重连一下自己 */ -- Gitee From 2bff479bb806ae2eb047febae004fe4edf0fe90f Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 2 Sep 2020 16:42:26 +0800 Subject: [PATCH 17/59] =?UTF-8?q?worker=E5=90=91dashboard=E7=9B=B4?= =?UTF-8?q?=E6=8E=A8=E7=83=ADkey?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/common/tool/flush/FlushUtil.java | 43 ++++++------------- .../netty/dashboard/DashboardHolder.java | 4 +- .../worker/netty/filter/HeartBeatFilter.java | 3 +- 3 files changed, 17 insertions(+), 33 deletions(-) diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java index af411d0..54a7e6d 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/flush/FlushUtil.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.common.tool.flush; -import com.jd.platform.hotkey.common.model.HotKeyMsg; import io.netty.buffer.ByteBuf; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; @@ -17,36 +16,20 @@ public class FlushUtil { * 往channel里输出消息 */ public static void flush(ChannelHandlerContext channelHandlerContext, ByteBuf byteBuf) { -// if (channelHandlerContext.channel().isWritable()) { -// channelHandlerContext.channel().writeAndFlush(byteBuf).addListener(future -> { -// if (!future.isSuccess()) { -// logger.warn("flush error " + future.cause().getMessage()); -// } -// }); -// } else { - try { - //同步发送 - channelHandlerContext.channel().writeAndFlush(byteBuf).sync(); - } catch (InterruptedException e) { - logger.error("flush error " + e.getMessage()); + if (channelHandlerContext.channel().isWritable()) { + channelHandlerContext.channel().writeAndFlush(byteBuf).addListener(future -> { + if (!future.isSuccess()) { + logger.warn("flush error " + future.cause().getMessage()); + } + }); + } else { + try { + //同步发送 + channelHandlerContext.channel().writeAndFlush(byteBuf).sync(); + } catch (InterruptedException e) { + logger.error("flush error " + e.getMessage()); + } } -// } } - public static void flush(ChannelHandlerContext channelHandlerContext, HotKeyMsg hotKeyMsg) { -// if (channelHandlerContext.channel().isWritable()) { -// channelHandlerContext.channel().writeAndFlush(hotKeyMsg).addListener(future -> { -// if (!future.isSuccess()) { -// logger.warn("flush error " + future.cause().getMessage()); -// } -// }); -// } else { - try { - //同步发送 - channelHandlerContext.channel().writeAndFlush(hotKeyMsg).sync(); - } catch (InterruptedException e) { - logger.error("flush error " + e.getMessage()); - } -// } - } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java index 6e0d57e..b3ea480 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/dashboard/DashboardHolder.java @@ -21,6 +21,8 @@ public class DashboardHolder { public static Channel channel = null; public static void flushToDashboard(String message) { - channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_HOT_KEY, message))); + HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.REQUEST_HOT_KEY); + hotKeyMsg.setBody(message); + channel.writeAndFlush(MsgBuilder.buildByteBuf(hotKeyMsg)); } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java index bc8171e..031c57f 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HeartBeatFilter.java @@ -2,7 +2,6 @@ package com.jd.platform.hotkey.worker.netty.filter; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.typeenum.MessageType; -import com.jd.platform.hotkey.common.tool.flush.FlushUtil; import io.netty.channel.ChannelHandlerContext; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; @@ -19,7 +18,7 @@ public class HeartBeatFilter implements INettyMsgFilter { @Override public boolean chain(HotKeyMsg message, ChannelHandlerContext ctx) { if (MessageType.PING == message.getMessageType()) { - FlushUtil.flush(ctx, new HotKeyMsg(MessageType.PONG)); + ctx.writeAndFlush(new HotKeyMsg(MessageType.PONG)); return false; } return true; -- Gitee From 534969926cc1d37563f9e839af5ee9295ae71865 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 17 Sep 2020 14:00:13 +0800 Subject: [PATCH 18/59] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/configcenter/ConfigConstant.java | 4 +++ .../hotkey/worker/config/DisruptorConfig.java | 33 ------------------- .../worker/keylistener/KeyListener.java | 8 ++++- .../hotkey/worker/starters/EtcdStarter.java | 30 +++++++++++++++++ 4 files changed, 41 insertions(+), 34 deletions(-) delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index 2a7eff0..1257d70 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -58,6 +58,10 @@ public interface ConfigConstant { * 存放客户端hotKey访问次数和总访问次数的path */ String keyHitCountPath = "/jd/keyHitCount/"; + /** + * 是否开启日志 + */ + String logToggle = "/jd/logOn"; /** * 清理历史数据的配置的path diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java deleted file mode 100644 index 0a92f8f..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/config/DisruptorConfig.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.jd.platform.hotkey.worker.config; - -import com.jd.platform.hotkey.worker.keylistener.IKeyListener; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -import javax.annotation.Resource; - -/** - * @author wuweifeng wrote on 2019-12-11 - * @version 1.0 - */ -@Configuration -public class DisruptorConfig { - @Resource - private IKeyListener iKeyListener; - - @Value("${open.timeout}") - private boolean openTimeOut; - @Value("${thread.count}") - private int threadCount; - -// @Bean -// public MessageProducer messageProducer() { -// InitConstant.openTimeOut = openTimeOut; -// //将实际值赋给static变量 -// InitConstant.threadCount = threadCount; -// -// return ProducerFactory.createHotKeyProducer(iKeyListener); -// } - - -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java b/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java index e941eb6..f944394 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/keylistener/KeyListener.java @@ -9,6 +9,7 @@ import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.worker.cache.CaffeineCacheHolder; import com.jd.platform.hotkey.worker.netty.pusher.IPusher; import com.jd.platform.hotkey.worker.rule.KeyRuleHolder; +import com.jd.platform.hotkey.worker.starters.EtcdStarter; import com.jd.platform.hotkey.worker.tool.SlidingWindow; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,7 +76,12 @@ public class KeyListener implements IKeyListener { //开启推送 hotKeyModel.setCreateTime(SystemClock.now()); - logger.info(NEW_KEY_EVENT + hotKeyModel.getKey()); + + //当开关打开时，打印日志。大促时关闭日志，就不打印了 + if (EtcdStarter.LOGGER_ON) { + logger.info(NEW_KEY_EVENT + hotKeyModel.getKey()); + } + //分别推送到各client和etcd for (IPusher pusher : iPushers) { pusher.push(hotKeyModel); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java index 2c60d6e..152af11 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java @@ -46,6 +46,11 @@ import static com.jd.platform.hotkey.worker.tool.InitConstant.*; */ @Component public class EtcdStarter { + /** + * 是否开启日志 + */ + public static boolean LOGGER_ON = true; + private Logger logger = LoggerFactory.getLogger(getClass()); @Resource @@ -100,6 +105,31 @@ public class EtcdStarter { return !DEFAULT_PATH.equals(workerPath); } + @PostConstruct + public void watchLog() { + AsyncPool.asyncDo(() -> { + try { + String loggerOn = configCenter.get(ConfigConstant.logToggle); + LOGGER_ON = "true".equals(loggerOn) || "1".equals(loggerOn); + } catch (StatusRuntimeException ex) { + logger.error(ETCD_DOWN); + } + + KvClient.WatchIterator watchIterator = configCenter.watch(ConfigConstant.logToggle); + while (watchIterator.hasNext()) { + WatchUpdate watchUpdate = watchIterator.next(); + List eventList = watchUpdate.getEvents(); + + KeyValue keyValue = eventList.get(0).getKv(); + logger.info("log toggle changed : " + keyValue); + + String value = keyValue.getValue().toStringUtf8(); + LOGGER_ON = "true".equals(value) || "1".equals(value); + } + }); + + } + /** * 启动回调监听器，监听rule变化 */ -- Gitee From b9f94a6c86932b26adb16819d3beb4f8e3f0aa7e Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 17 Sep 2020 15:04:10 +0800 Subject: [PATCH 19/59] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/pom.xml | 8 -------- .../hotkey/common/configcenter/ConfigConstant.java | 6 ++++++ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/common/pom.xml b/common/pom.xml index 8b30a03..407feb5 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -15,7 +15,6 @@ 1.8 4.1.42.Final - 3.4.2 1.1.7.3 1.2.70 0.0.16 @@ -53,13 +52,6 @@ netty-all ${netty4.version} - - - com.lmax - disruptor - ${disruptor.version} - - com.alibaba fastjson diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index 1257d70..009a107 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -68,6 +68,12 @@ public interface ConfigConstant { * time unit : day */ String clearCfgPath = "/jd/clearCfg/"; + + /** + * app配置 + */ + String appCfgPath = "/jd/appCfg/"; + /** * 控制台启动的netty端口 */ -- Gitee From 191ff79c64ecd9f106e8407bdb6a52186bfaa321 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 17 Sep 2020 15:12:05 +0800 Subject: [PATCH 20/59] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jd/platform/hotkey/worker/cache/Test.java | 64 ------------------- .../hotkey/worker/model/ClientInfo.java | 62 ------------------ .../worker/netty/filter/KeyCounterFilter.java | 1 - .../netty/server/NodesServerHandler.java | 1 - worker/src/main/resources/application.yml | 1 - 5 files changed, 129 deletions(-) delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/cache/Test.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/Test.java b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/Test.java deleted file mode 100644 index bf2afa4..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/Test.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.jd.platform.hotkey.worker.cache; - -import com.github.benmanes.caffeine.cache.Cache; -import com.github.benmanes.caffeine.cache.Caffeine; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-05-04 - */ -@Component -public class Test { - @Resource(name = "hotKeyCache") - private Cache hotCache; - - public static void main(String[] args) { - Map map = new HashMap(); -// map.put("a", "b"); -// System.out.println(map); - Cache cache = Caffeine.newBuilder() - .initialCapacity(1024)//初始大小 - .maximumSize(5000000)//最大数量 - .expireAfterWrite(5, TimeUnit.MINUTES)//过期时间 - .softValues() - .build(); - - cache.put("yuan2012555", 1); -// System.out.println(cache.getIfPresent("yuan2012555")); - - - System.out.println("lwymail163".hashCode() %4); - System.out.println("272551766_m".hashCode() %4); - System.out.println("hanxu123".hashCode() %4); - System.out.println("abc123ab".hashCode() %4); - - System.out.println("hanxu123hanxu".hashCode() %4); - System.out.println(Math.abs("lwy163mail".hashCode() %4)); - - } - -// @PostConstruct -// public void aa() throws InterruptedException { -// Executor executor = Executors.newCachedThreadPool(); -// Cache cache = Caffeine.newBuilder() -// .executor(executor) -// .initialCapacity(1024)//初始大小 -// .maximumSize(5000000)//最大数量 -// .expireAfterWrite(5, TimeUnit.MINUTES)//过期时间 -// .softValues() -// .build(); -// long i = 0; -// while (true) { -// cache.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); -//// hotCache.put("i" + i, i); -// i++; -// } -// } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java b/worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java deleted file mode 100644 index 081652b..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/model/ClientInfo.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.jd.platform.hotkey.worker.model; - -import io.netty.channel.ChannelHandlerContext; - -/** - * @author wuweifeng wrote on 2019-12-05 - * @version 1.0 - */ -public class ClientInfo { - /** - * 应用名 - */ - private String appName; - /** - * 连接的id - */ - private String channelId; - - private ChannelHandlerContext channelHandlerContext; - - public ClientInfo() { - } - - public ClientInfo(String appName, String channelId, ChannelHandlerContext channelHandlerContext) { - this.appName = appName; - this.channelId = channelId; - this.channelHandlerContext = channelHandlerContext; - } - - @Override - public String toString() { - return "ClientInfo{" + - "appName='" + appName + '\'' + - ", channelId='" + channelId + '\'' + - ", channelHandlerContext=" + channelHandlerContext + - '}'; - } - - public ChannelHandlerContext getChannelHandlerContext() { - return channelHandlerContext; - } - - public void setChannelHandlerContext(ChannelHandlerContext channelHandlerContext) { - this.channelHandlerContext = channelHandlerContext; - } - - public String getAppName() { - return appName; - } - - public void setAppName(String appName) { - this.appName = appName; - } - - public String getChannelId() { - return channelId; - } - - public void setChannelId(String channelId) { - this.channelId = channelId; - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java index 115200f..0645822 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java @@ -53,7 +53,6 @@ public class KeyCounterFilter implements INettyMsgFilter { private void publishMsg(String appName, HotKeyMsg message, ChannelHandlerContext ctx) { - //老版的用的单个HotKeyModel，新版用的数组 List models = message.getKeyCountModels(); if (CollectionUtil.isEmpty(models)) { return; diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java index 658b4c0..df3debd 100755 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/server/NodesServerHandler.java @@ -40,7 +40,6 @@ public class NodesServerHandler extends SimpleChannelInboundHandler { return; } } - } @Override diff --git a/worker/src/main/resources/application.yml b/worker/src/main/resources/application.yml index 3d33600..1708974 100644 --- a/worker/src/main/resources/application.yml +++ b/worker/src/main/resources/application.yml @@ -5,7 +5,6 @@ netty: local: address: ${localAddress:} #有些获取到的ip不能用，需要手工配worker的地址 open: - timeout: ${openTimeOut:true} monitor: ${openMonitor:false} #开启持续无key发送监控，如果持续1分钟没发来key，就断开和etcd的连接，之后重建和客户端连接 thread: count: ${threadCount:0} -- Gitee From ff7f5e8d66882bd676262f635eac26188612ca08 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 17 Sep 2020 15:49:15 +0800 Subject: [PATCH 21/59] merge erp --- .../dashboard/DashboardApplication.java | 13 +- .../biz/controller/AppCfgController.java | 58 ++++ .../controller/ChangLogController.java | 4 +- .../{ => biz}/controller/ClearController.java | 4 +- .../{ => biz}/controller/KeyController.java | 27 +- .../{ => biz}/controller/RuleController.java | 8 +- .../{ => biz}/controller/UserController.java | 52 ++-- .../controller/WorkerController.java | 38 +-- .../{ => biz}/mapper/ChangeLogMapper.java | 2 +- .../{ => biz}/mapper/KeyRecordMapper.java | 2 +- .../{ => biz}/mapper/RulesMapper.java | 2 +- .../{ => biz}/mapper/StatisticsMapper.java | 2 +- .../{ => biz}/mapper/SummaryMapper.java | 2 +- .../{ => biz}/mapper/UserMapper.java | 2 +- .../{ => biz}/mapper/WorkerMapper.java | 2 +- .../dashboard/biz/service/AppCfgService.java | 21 ++ .../{ => biz}/service/ChangeLogService.java | 2 +- .../{ => biz}/service/ClearService.java | 2 +- .../{ => biz}/service/KeyService.java | 2 +- .../{ => biz}/service/RuleService.java | 2 +- .../{ => biz}/service/UserService.java | 2 +- .../{ => biz}/service/WorkerService.java | 2 +- .../biz/service/impl/AppCfgServiceImpl.java | 82 ++++++ .../service/impl/ChangeLogServiceImpl.java | 4 +- .../service/impl/ClearServiceImpl.java | 7 +- .../service/impl/KeyServiceImpl.java | 6 +- .../service/impl/RuleServiceImpl.java | 8 +- .../service/impl/UserServiceImpl.java | 4 +- .../service/impl/WorkerServiceImpl.java | 4 +- .../dashboard/common/base/BaseController.java | 3 +- .../dashboard/common/domain/Constant.java | 1 + .../common/domain/PushMsgWrapper.java | 48 ++++ .../dashboard/common/domain/vo/AppCfgVo.java | 131 +++++++++ .../common/domain/vo/ClearCfgVo.java | 4 +- .../common/domain/vo/HotKeyLineChartVo.java | 3 +- .../common/domain/vo/LineChartVo.java | 3 +- .../common/domain/vo/RuleLineChartVo.java | 4 +- .../dashboard/common/monitor/DataHandler.java | 67 ++++- .../dashboard/common/monitor/PushHandler.java | 138 +++++++++ .../common/monitor/SlidingWindow.java | 136 +++++++++ .../dashboard/controller/TestController.java | 21 -- .../dashboard/erp/ErpUimInterceptor.java | 2 +- .../{common/config => etcd}/EtcdConfig.java | 2 +- .../hotkey/dashboard/etcd/EtcdMonitor.java | 272 ++++++++++++++++++ .../dashboard/interceptor/JwtInterceptor.java | 2 - .../hotkey/dashboard/util/CommonUtil.java | 51 +--- dashboard/src/main/resources/application.yml | 18 +- .../dashboard/mapper/ChangeLogMapper.xml | 6 +- .../dashboard/mapper/KeyRecordMapper.xml | 2 +- .../hotkey/dashboard/mapper/RulesMapper.xml | 2 +- .../dashboard/mapper/StatisticsMapper.xml | 10 +- .../hotkey/dashboard/mapper/SummaryMapper.xml | 4 +- .../hotkey/dashboard/mapper/UserMapper.xml | 4 +- .../hotkey/dashboard/mapper/WorkerMapper.xml | 4 +- dashboard/src/main/resources/db.sql | 5 + .../admin/assets/js/bootstrap/js/base_list.js | 2 +- .../static/admin/common/js/appcfg-edit.js | 25 ++ .../templates/admin/appcfg/edit.html | 89 ++++++ .../templates/admin/appcfg/list.html | 166 +++++++++++ .../templates/admin/common/html/leftMenu.html | 62 ++-- .../main/resources/templates/admin/main.html | 18 +- 61 files changed, 1412 insertions(+), 259 deletions(-) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/AppCfgController.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/ChangLogController.java (90%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/ClearController.java (93%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/KeyController.java (90%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/RuleController.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/UserController.java (84%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/controller/WorkerController.java (73%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/ChangeLogMapper.java (87%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/KeyRecordMapper.java (92%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/RulesMapper.java (83%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/StatisticsMapper.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/SummaryMapper.java (91%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/UserMapper.java (91%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/mapper/WorkerMapper.java (91%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/AppCfgService.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/ChangeLogService.java (91%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/ClearService.java (90%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/KeyService.java (95%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/RuleService.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/UserService.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/WorkerService.java (93%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/ChangeLogServiceImpl.java (90%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/ClearServiceImpl.java (93%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/KeyServiceImpl.java (98%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/RuleServiceImpl.java (94%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/UserServiceImpl.java (97%) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{ => biz}/service/impl/WorkerServiceImpl.java (97%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/TestController.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/{common/config => etcd}/EtcdConfig.java (93%) create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java create mode 100644 dashboard/src/main/resources/static/admin/common/js/appcfg-edit.js create mode 100644 dashboard/src/main/resources/templates/admin/appcfg/edit.html create mode 100644 dashboard/src/main/resources/templates/admin/appcfg/list.html diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 3070702..e1431c9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -1,9 +1,6 @@ package com.jd.platform.hotkey.dashboard; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.scheduling.annotation.EnableAsync; @@ -13,10 +10,7 @@ import org.springframework.scheduling.annotation.EnableScheduling; @EnableAsync @EnableScheduling @SpringBootApplication -public class DashboardApplication implements CommandLineRunner { - - private Logger logger = LoggerFactory.getLogger(getClass()); - +public class DashboardApplication{ public static void main(String[] args) { try { @@ -26,9 +20,4 @@ public class DashboardApplication implements CommandLineRunner { } } - - @Override - public void run(String... args) { - } - } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/AppCfgController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/AppCfgController.java new file mode 100644 index 0000000..75306ca --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/AppCfgController.java @@ -0,0 +1,58 @@ +package com.jd.platform.hotkey.dashboard.biz.controller; + +import com.jd.platform.hotkey.dashboard.common.base.BaseController; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.Result; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; +import com.jd.platform.hotkey.dashboard.biz.service.AppCfgService; +import org.springframework.stereotype.Controller; +import org.springframework.ui.ModelMap; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; + + +@Controller +@RequestMapping("/appCfg") +public class AppCfgController extends BaseController { + + + private String prefix = "admin/appcfg"; + + + @Resource + private AppCfgService appCfgService; + + @GetMapping("/view") + public String view(ModelMap modelMap){ + modelMap.put("title", Constant.APP_CFG_VIEW); + return prefix + "/list"; + } + + + @PostMapping("/list") + @ResponseBody + public Page list(PageReq page, String app){ + return appCfgService.pageAppCfgVo(page, app); + } + + + @GetMapping("/edit/{app}") + public String edit(@PathVariable("app") String app, ModelMap modelMap){ + modelMap.put("appCfg", appCfgService.selectAppCfgVo(app)); + return prefix + "/edit"; + } + + @PostMapping("/save") + @ResponseBody + public Result save(AppCfgVo cfg){ + cfg.setModifier(userName()); + appCfgService.saveAppCfgVo(cfg); + return Result.success(); + } + + +} + diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ChangLogController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ChangLogController.java similarity index 90% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ChangLogController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ChangLogController.java index e9a1788..1e18dc2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ChangLogController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ChangLogController.java @@ -1,11 +1,11 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.*; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.model.ChangeLog; -import com.jd.platform.hotkey.dashboard.service.ChangeLogService; +import com.jd.platform.hotkey.dashboard.biz.service.ChangeLogService; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ClearController.java similarity index 93% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ClearController.java index 0ce81f0..e6ce6b4 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/ClearController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/ClearController.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.Constant; @@ -6,7 +6,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.Result; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; -import com.jd.platform.hotkey.dashboard.service.ClearService; +import com.jd.platform.hotkey.dashboard.biz.service.ClearService; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java similarity index 90% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java index f3f362d..edf8de7 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; @@ -14,7 +14,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; -import com.jd.platform.hotkey.dashboard.service.KeyService; +import com.jd.platform.hotkey.dashboard.biz.service.KeyService; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.ExcelUtil; import org.springframework.stereotype.Controller; @@ -34,7 +34,7 @@ import java.util.List; @Controller @RequestMapping("/key") public class KeyController extends BaseController { - + private String prefix = "admin/key"; @Resource @@ -99,9 +99,9 @@ public class KeyController extends BaseController { @GetMapping("/add") - public String add(){ - return prefix + "/add"; - } + public String add(){ + return prefix + "/add"; + } @PostMapping("/add") @ResponseBody @@ -122,12 +122,19 @@ public class KeyController extends BaseController { } - @PostMapping("/edit") - @ResponseBody - public Result editSave(KeyTimely key) { + /*@GetMapping("/edit/{id}") + public String edit(@PathVariable("id") Long id, ModelMap modelMap){ + modelMap.put("key", keyService.selectByPk(id)); + return prefix + "/edit"; + }*/ + + + @PostMapping("/edit") + @ResponseBody + public Result editSave(KeyTimely key) { checkApp(key.getAppName()); return Result.success(keyService.updateKeyByUser(key)); - } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java index e1869c9..ce8600d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/RuleController.java @@ -1,9 +1,8 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import cn.hutool.json.JSONArray; import cn.hutool.json.JSONObject; import cn.hutool.json.JSONTokener; -import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.Constant; @@ -14,9 +13,8 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; -import com.jd.platform.hotkey.dashboard.model.Rule; import com.jd.platform.hotkey.dashboard.model.Rules; -import com.jd.platform.hotkey.dashboard.service.RuleService; +import com.jd.platform.hotkey.dashboard.biz.service.RuleService; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; @@ -31,7 +29,7 @@ import java.util.List; @Controller @RequestMapping("/rule") public class RuleController extends BaseController { - + @Resource private RuleService ruleService; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java similarity index 84% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java index bc571ba..0f84c8e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/UserController.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import java.util.*; @@ -11,7 +11,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.Result; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import io.jsonwebtoken.Claims; @@ -100,10 +100,10 @@ public class UserController extends BaseController { @GetMapping("/view") - public String view(ModelMap modelMap){ + public String view(ModelMap modelMap){ modelMap.put("title", Constant.USER_MANAGE_VIEW); - return "admin/user/list"; - } + return "admin/user/list"; + } @PostMapping("/list") @@ -114,10 +114,10 @@ public class UserController extends BaseController { } - @GetMapping("/add") - public String add(){ - return "admin/user/add"; - } + @GetMapping("/add") + public String add(){ + return "admin/user/add"; + } @PostMapping("/add") @ResponseBody @@ -135,31 +135,31 @@ public class UserController extends BaseController { @GetMapping("/edit/{id}") - public String edit(@PathVariable("id") Integer id, ModelMap modelMap){ + public String edit(@PathVariable("id") Integer id, ModelMap modelMap){ modelMap.put("user", userService.selectByPrimaryKey(id)); - return "admin/user/edit"; - } + return "admin/user/edit"; + } - @PostMapping("/edit") - @ResponseBody - public Result editSave(User user) { - return Result.success(userService.updateUser(user)); - } + @PostMapping("/edit") + @ResponseBody + public Result editSave(User user) { + return Result.success(userService.updateUser(user)); + } @GetMapping("/editPwd/{id}") - public String editPwd(@PathVariable("id") Integer id, ModelMap modelMap){ + public String editPwd(@PathVariable("id") Integer id, ModelMap modelMap){ modelMap.put("user", userService.selectByPrimaryKey(id)); - return "admin/user/editPwd"; - } - - @PostMapping("/editPwd") - @ResponseBody - public Result editPwdSave(User user){ - return Result.success(userService.updateUser(user)); - } + return "admin/user/editPwd"; + } + + @PostMapping("/editPwd") + @ResponseBody + public Result editPwdSave(User user){ + return Result.success(userService.updateUser(user)); + } @GetMapping("Out404") diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/WorkerController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java similarity index 73% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/WorkerController.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java index 4e9471e..f8ab766 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/WorkerController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/WorkerController.java @@ -1,11 +1,11 @@ -package com.jd.platform.hotkey.dashboard.controller; +package com.jd.platform.hotkey.dashboard.biz.controller; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.*; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.model.Worker; -import com.jd.platform.hotkey.dashboard.service.WorkerService; +import com.jd.platform.hotkey.dashboard.biz.service.WorkerService; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; @@ -16,31 +16,31 @@ import javax.annotation.Resource; @Controller @RequestMapping("/worker") public class WorkerController extends BaseController { - + private String prefix = "admin/worker"; @Resource private WorkerService workerService; @GetMapping("/view") - public String view(ModelMap modelMap){ + public String view(ModelMap modelMap){ modelMap.put("title", Constant.WORKER_VIEW); - return prefix + "/list"; - } + return prefix + "/list"; + } @PostMapping("/list") @ResponseBody - public Page list(PageReq page, String searchText){ + public Page list(PageReq page, String searchText){ PageInfo info = workerService.pageWorker(page, param(searchText)); return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); } - @GetMapping("/add") - public String add(){ - return prefix + "/add"; - } + @GetMapping("/add") + public String add(){ + return prefix + "/add"; + } @PostMapping("/add") @ResponseBody @@ -59,18 +59,18 @@ public class WorkerController extends BaseController { @GetMapping("/edit/{key}") - public String edit(@PathVariable("key") String key, ModelMap modelMap){ + public String edit(@PathVariable("key") String key, ModelMap modelMap){ modelMap.put("worker", workerService.selectByKey(key.replace("_","/"))); - return prefix + "/edit"; - } - + return prefix + "/edit"; + } - @PostMapping("/edit") - @ResponseBody - public Result editSave(Worker worker) { + + @PostMapping("/edit") + @ResponseBody + public Result editSave(Worker worker) { worker.setUpdateUser(userName()); return Result.success(workerService.updateWorkerByUser(worker)); - } + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.java similarity index 87% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.java index c36f871..a1ceb20 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.ChangeLog; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java similarity index 92% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java index e135ff6..dde2c25 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.KeyRecord; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.java similarity index 83% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.java index 1c6835c..3c1c894 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.model.Rules; import org.apache.ibatis.annotations.Mapper; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.java index daf848e..a657342 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.Statistics; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.java similarity index 91% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.java index 0305c4c..99882f7 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java similarity index 91% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java index c8d72ee..257a27d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.User; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.java similarity index 91% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.java index 501316b..29befc2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.mapper; +package com.jd.platform.hotkey.dashboard.biz.mapper; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.Worker; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/AppCfgService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/AppCfgService.java new file mode 100644 index 0000000..fa814b8 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/AppCfgService.java @@ -0,0 +1,21 @@ +package com.jd.platform.hotkey.dashboard.biz.service; + +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; + +/** + * @ProjectName: hotkey + * @ClassName: ClearService + * @Author: liyunfeng31 + * @Date: 2020/8/3 9:51 + */ +public interface AppCfgService { + + Page pageAppCfgVo(PageReq page, String app); + + AppCfgVo selectAppCfgVo(String app); + + void saveAppCfgVo(AppCfgVo cfg); + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ChangeLogService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ChangeLogService.java similarity index 91% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ChangeLogService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ChangeLogService.java index 498f4e4..cfed249 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ChangeLogService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ChangeLogService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ClearService.java similarity index 90% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ClearService.java index 5266ef3..383fa9c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/ClearService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/ClearService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java similarity index 95% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java index 558e814..d8908f9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.Page; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/RuleService.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/RuleService.java index 5580df1..1a55b99 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/RuleService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/UserService.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/UserService.java index 71f57ea..b4bb868 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/UserService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/WorkerService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/WorkerService.java similarity index 93% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/WorkerService.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/WorkerService.java index 4995172..398dae8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/WorkerService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/WorkerService.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service; +package com.jd.platform.hotkey.dashboard.biz.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java new file mode 100644 index 0000000..74edd41 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -0,0 +1,82 @@ +package com.jd.platform.hotkey.dashboard.biz.service.impl; + +import com.alibaba.fastjson.JSON; +import com.github.pagehelper.util.StringUtil; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.biz.service.AppCfgService; +import com.jd.platform.hotkey.dashboard.common.domain.Page; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; +import com.jd.platform.hotkey.dashboard.common.monitor.PushHandler; +import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; +import com.jd.platform.hotkey.dashboard.util.PageUtil; +import org.springframework.stereotype.Service; +import org.springframework.util.StringUtils; + +import javax.annotation.Resource; +import java.util.ArrayList; +import java.util.List; + +/** + * @ProjectName: hotkey + * @ClassName: AppCfgServiceImpl + * @Author: liyunfeng31 + * @Date: 2020/9/2 9:57 + */ +@Service +public class AppCfgServiceImpl implements AppCfgService { + + + @Resource + private IConfigCenter configCenter; + + + @Override + public Page pageAppCfgVo(PageReq page, String app) { + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + List cfgVos = new ArrayList<>(); + for (KeyValue kv : keyValues) { + String v = kv.getValue().toStringUtf8(); + String key = kv.getKey().toStringUtf8(); + if(StringUtil.isEmpty(v)){ + configCenter.put(key, JSON.toJSONString(new AppCfgVo(key))); + continue; + } + AppCfgVo vo = JSON.parseObject(v, AppCfgVo.class); + vo.setVersion(kv.getModRevision()); + String k = key.replace(ConfigConstant.appCfgPath,""); + if(StringUtils.isEmpty(app)){ + cfgVos.add(vo); + }else{ + if(k.equals(app)){ cfgVos.add(vo); } + } + } + return PageUtil.pagination(cfgVos,page.getPageSize(),page.getPageNum()-1); + } + + @Override + public AppCfgVo selectAppCfgVo(String app) { + KeyValue kv = configCenter.getKv(ConfigConstant.appCfgPath + app); + if(kv == null || kv.getValue() == null){ + AppCfgVo ap = new AppCfgVo(app); + configCenter.put(ConfigConstant.appCfgPath + app, JSON.toJSONString(ap)); + return ap; + } + String v = kv.getValue().toStringUtf8(); + AppCfgVo cfg = JSON.parseObject(v, AppCfgVo.class); + cfg.setVersion(kv.getModRevision()); + return cfg; + } + + /** + * todo 多节点问题 待完善 + */ + @Override + public void saveAppCfgVo(AppCfgVo cfg) { + cfg.setWindow(new SlidingWindow(cfg.getWarnPeriod()/60,cfg.getWarnThreshold())); + PushHandler.appCfgMap.put(cfg.getApp(), cfg); + configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ChangeLogServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ChangeLogServiceImpl.java similarity index 90% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ChangeLogServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ChangeLogServiceImpl.java index 50e11af..216bb79 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ChangeLogServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ChangeLogServiceImpl.java @@ -1,12 +1,12 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; +import com.jd.platform.hotkey.dashboard.biz.service.ChangeLogService; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.model.ChangeLog; -import com.jd.platform.hotkey.dashboard.service.ChangeLogService; import org.springframework.stereotype.Service; import javax.annotation.Resource; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ClearServiceImpl.java similarity index 93% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ClearServiceImpl.java index 8227a95..cd5d872 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/ClearServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/ClearServiceImpl.java @@ -1,18 +1,19 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.biz.service.ClearService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.ClearCfgVo; -import com.jd.platform.hotkey.dashboard.service.ClearService; -import com.jd.platform.hotkey.dashboard.service.UserService; import org.apache.commons.collections4.CollectionUtils; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; + import javax.annotation.Resource; import java.util.ArrayList; import java.util.List; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java similarity index 98% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java index 09dbd32..43f7b84 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import cn.hutool.core.date.SystemClock; @@ -9,6 +9,8 @@ import com.ibm.etcd.api.Event; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.dashboard.biz.service.KeyService; +import com.jd.platform.hotkey.dashboard.biz.service.RuleService; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; @@ -23,8 +25,6 @@ import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; -import com.jd.platform.hotkey.dashboard.service.KeyService; -import com.jd.platform.hotkey.dashboard.service.RuleService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.PageUtil; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java similarity index 94% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java index 9dd493e..ec2f783 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import cn.hutool.core.lang.UUID; import com.alibaba.fastjson.JSON; @@ -8,14 +8,16 @@ import com.github.pagehelper.util.StringUtil; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.biz.service.RuleService; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.mapper.RulesMapper; import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; -import com.jd.platform.hotkey.dashboard.model.*; -import com.jd.platform.hotkey.dashboard.service.RuleService; +import com.jd.platform.hotkey.dashboard.model.ChangeLog; +import com.jd.platform.hotkey.dashboard.model.Rule; +import com.jd.platform.hotkey.dashboard.model.Rules; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/UserServiceImpl.java similarity index 97% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/UserServiceImpl.java index c03ea2e..2c068b9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/UserServiceImpl.java @@ -1,16 +1,16 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; import com.jd.platform.hotkey.dashboard.mapper.UserMapper; import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.service.UserService; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/WorkerServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/WorkerServiceImpl.java similarity index 97% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/WorkerServiceImpl.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/WorkerServiceImpl.java index 4c133ee..dac5232 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/WorkerServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/WorkerServiceImpl.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.service.impl; +package com.jd.platform.hotkey.dashboard.biz.service.impl; import cn.hutool.core.date.SystemClock; import com.alibaba.fastjson.JSON; @@ -7,6 +7,7 @@ import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.dashboard.biz.service.WorkerService; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; @@ -14,7 +15,6 @@ import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.mapper.WorkerMapper; import com.jd.platform.hotkey.dashboard.model.ChangeLog; import com.jd.platform.hotkey.dashboard.model.Worker; -import com.jd.platform.hotkey.dashboard.service.WorkerService; import org.springframework.dao.DuplicateKeyException; import org.springframework.stereotype.Service; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java index 7aa346f..e4065ec 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java @@ -8,8 +8,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; -import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import io.jsonwebtoken.Claims; import org.springframework.beans.propertyeditors.CustomDateEditor; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java index 952aa2a..d47ca2e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java @@ -37,6 +37,7 @@ public class Constant { public static final String CLEAR_VIEW = "数据清理"; + public static final String APP_CFG_VIEW = "应用配置"; public static final int MAX_DAY_RANGE = 3; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java new file mode 100644 index 0000000..937baf5 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/PushMsgWrapper.java @@ -0,0 +1,48 @@ +package com.jd.platform.hotkey.dashboard.common.domain; + +import cn.hutool.core.date.SystemClock; + +import java.io.Serializable; + +/** + * 报警消息包装类，用于保存事件最准确的时间 + */ +public class PushMsgWrapper implements Serializable { + + private String app; + + private Long date; + + private String msg; + + public PushMsgWrapper(String app) { + this.app = app; + this.date = SystemClock.now(); + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public Long getDate() { + return date; + } + + public void setDate(Long date) { + this.date = date; + } + + public String getMsg() { + return msg; + } + + public void setMsg(String msg) { + this.msg = msg; + } +} + + diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java new file mode 100644 index 0000000..8124945 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java @@ -0,0 +1,131 @@ +package com.jd.platform.hotkey.dashboard.common.domain.vo; + +import com.alibaba.fastjson.JSON; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; + +import java.beans.Transient; +import java.io.Serializable; + +/** + * @ProjectName: hotkey + * @ClassName: AppCfgVo + * @Author: liyunfeng31 + * @Date: 2020/9/2 10:29 + */ +public class AppCfgVo implements Serializable { + + private String app; + + /** + * 数据保存时长 + */ + private Integer dataTtl; + + /** + * 警报周期 + */ + private Integer warnPeriod; + + /** + * 警报阈值 + */ + private Integer warnThreshold; + + /** + * 警报开关 1开启 0关闭 + */ + private Integer warn; + + /** + * 版本 + */ + private Long version; + + /* + * 最后修改人 + */ + private String modifier; + + + private SlidingWindow window; + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public Integer getDataTtl() { + return dataTtl; + } + + public void setDataTtl(Integer dataTtl) { + this.dataTtl = dataTtl; + } + + public Integer getWarnPeriod() { + return warnPeriod; + } + + public void setWarnPeriod(Integer warnPeriod) { + this.warnPeriod = warnPeriod; + } + + public Integer getWarnThreshold() { + return warnThreshold; + } + + public void setWarnThreshold(Integer warnThreshold) { + this.warnThreshold = warnThreshold; + } + + public Integer getWarn() { + return warn; + } + + public void setWarn(Integer warn) { + this.warn = warn; + } + + public Long getVersion() { + return version; + } + + public void setVersion(Long version) { + this.version = version; + } + + public String getModifier() { + return modifier; + } + + public void setModifier(String modifier) { + this.modifier = modifier; + } + + public SlidingWindow getWindow() { + return window; + } + + public void setWindow(SlidingWindow window) { + this.window = window; + } + + public AppCfgVo() { + } + + public AppCfgVo(String app) { + this.app = app; + this.dataTtl = 30; + this.warnPeriod = 10*60; + this.warnThreshold = 1000; + this.version = 0L; + this.warn = 1; + this.modifier = "SYSTEM"; + this.window = new SlidingWindow(warnPeriod/60,warnThreshold); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java index 14b872b..65df755 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/ClearCfgVo.java @@ -1,5 +1,7 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import java.io.Serializable; + /** * @ProjectName: hotkey * @ClassName: ClearCfgVo @@ -7,7 +9,7 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; * @Author: liyunfeng31 * @Date: 2020/8/3 9:54 */ -public class ClearCfgVo { +public class ClearCfgVo implements Serializable { private String app; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java index 3392fc2..71171a6 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HotKeyLineChartVo.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Set; @@ -7,7 +8,7 @@ import java.util.Set; /** * @author liyunfeng31 */ -public class HotKeyLineChartVo { +public class HotKeyLineChartVo implements Serializable { private List xAxis; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java index 273ce59..3179c5f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import java.io.Serializable; import java.util.List; import java.util.Map; import java.util.Set; @@ -7,7 +8,7 @@ import java.util.Set; /** * @author liyunfeng31 */ -public class LineChartVo { +public class LineChartVo implements Serializable { private Set legend; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java index 31ba530..8f6fbc5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java @@ -1,9 +1,11 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import java.io.Serializable; + /** * @author liyunfeng31 */ -public class RuleLineChartVo { +public class RuleLineChartVo implements Serializable { private String rule; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index bd3a5e0..e2f6279 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -7,14 +7,17 @@ import com.google.common.collect.Queues; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.dashboard.biz.mapper.KeyRecordMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.StatisticsMapper; +import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.IRecord; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; -import com.jd.platform.hotkey.dashboard.mapper.StatisticsMapper; -import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.Statistics; +import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import org.slf4j.Logger; @@ -39,6 +42,7 @@ public class DataHandler { @Resource private KeyRecordMapper keyRecordMapper; + @Resource private StatisticsMapper statisticsMapper; @@ -48,17 +52,25 @@ public class DataHandler { @Resource private IConfigCenter configCenter; + @Resource + private PushHandler pushHandler; + + + private static final Integer CACHE_SIZE = 10000; + + /** * 队列 */ - private BlockingQueue queue = new LinkedBlockingQueue<>(); + private BlockingQueue RECORD_QUEUE = new LinkedBlockingQueue<>(); + /** * 入队 */ public void offer(IRecord record) { try { - queue.put(record); + RECORD_QUEUE.put(record); } catch (InterruptedException e) { e.printStackTrace(); } @@ -68,7 +80,7 @@ public class DataHandler { while (true) { try { List records = new ArrayList<>(); - Queues.drain(queue, records, 1000, 1, TimeUnit.SECONDS); + Queues.drain(RECORD_QUEUE, records, CACHE_SIZE, 1, TimeUnit.SECONDS); if (CollectionUtil.isEmpty(records)) { continue; } @@ -204,6 +216,7 @@ public class DataHandler { } + /** * 每天根据app的配置清理过期数据 */ @@ -211,12 +224,12 @@ public class DataHandler { public void clearExpireData() { try { LocalDateTime now = LocalDateTime.now(); - List keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); for (KeyValue kv : keyValues) { - String key = kv.getKey().toStringUtf8(); - String ttl = kv.getValue().toStringUtf8(); - String app = key.replace(ConfigConstant.clearCfgPath, ""); - Date expireDate = DateUtil.ldtToDate(now.minusDays(Integer.parseInt(ttl))); + String val = kv.getValue().toStringUtf8(); + AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); + String app = cfg.getApp(); + Date expireDate = DateUtil.ldtToDate(now.minusDays(cfg.getDataTtl())); summaryMapper.clearExpireData(app, expireDate); keyRecordMapper.clearExpireData(app, expireDate); statisticsMapper.clearExpireData(app, expireDate); @@ -228,4 +241,36 @@ public class DataHandler { } + public void dealHotKey() { + while (true) { + try { + //获取发来的这个热key，存入本地caffeine，设置过期时间 + HotKeyModel model = HotKeyReceiver.take(); + //将该key放入实时热key本地缓存中 + if(model != null){ + HotKeyReceiver.put(model); + putRecord(model.getAppName(), model.getKey()); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + + public void putRecord(String app, String key) throws InterruptedException { + this.offer(new IRecord() { + @Override + public String appNameKey() { return app + "/" + key; } + @Override + public String value() { return UUID.randomUUID().toString(); } + @Override + public int type() { return 0; } + @Override + public Date createTime() { return new Date(); } + }); + // 监控和推送 + pushHandler.monitorAndPush(app); + } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java new file mode 100644 index 0000000..073cd9f --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java @@ -0,0 +1,138 @@ +package com.jd.platform.hotkey.dashboard.common.monitor; + +import com.alibaba.fastjson.JSON; +import com.ibm.etcd.api.KeyValue; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; +import com.jd.platform.hotkey.dashboard.common.domain.PushMsgWrapper; +import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * @ProjectName: hotkey + * @ClassName: PushHandler + * @Description: 处理推送 + * @Author: liyunfeng31 + * @Date: 2020/9/3 10:47 + */ + +@Component +public class PushHandler { + + /** + * 拦截重复报警间隔 10分钟 + */ + private static final long interval = 10*60*1000L; + + /** + * app-config map + */ + public static Map appCfgMap = new ConcurrentHashMap<>(); + + /** + * app-time 用于app存储报警时间 做拦截 + */ + private static Map appIntervalMap = new ConcurrentHashMap<>(); + + /** + * 超过阈值的热点记录次数 放到队列里 + */ + private static final BlockingQueue MSG_QUEUE = new LinkedBlockingQueue<>(); + + @Resource + private IConfigCenter configCenter; + @Resource + private UserService userService; + + + /** + * 监控和入队 + */ + public void monitorAndPush(String app) throws InterruptedException { + AppCfgVo cfg = PushHandler.appCfgMap.get(app); + if(cfg.getWarn().equals(1)){ + SlidingWindow wd = cfg.getWindow(); + int count = wd.addCount(1); + if(count > 0){ + MSG_QUEUE.put(new PushMsgWrapper(app)); + } + } + } + + + /** + * 启动线程处理警报消息队列 + */ + public void pushWarnMsg() { + // 初始化MAP到内存 + initAppCfgMap(); + while (true){ + try { + PushMsgWrapper msgWrapper = MSG_QUEUE.take(); + String warnApp = msgWrapper.getApp(); + Long msgTime = msgWrapper.getDate(); + boolean send = check(warnApp, msgTime); + if(send){ doPush(); } + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + + /** + * 防止重复发送警报 + * @param warnApp app + * @param msgTime time + * @return result + */ + private synchronized boolean check(String warnApp, Long msgTime){ + Long maxTime = appIntervalMap.get(warnApp); + if(maxTime == null){ + appIntervalMap.put(warnApp,msgTime+interval); + return true; + }else{ + if(msgTime > maxTime){ + appIntervalMap.put(warnApp,msgTime+interval); + return true; + } + } + return false; + } + + // todo 执行报警 + private static void doPush() { + } + + + /** + * 初始化cfgMap和滑动窗口 + */ + private void initAppCfgMap() { + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + if(CollectionUtils.isEmpty(keyValues)){ + List apps = userService.listApp(); + for (String ap : apps) { + AppCfgVo cfg = new AppCfgVo(ap); + appCfgMap.put(ap,cfg); + configCenter.put(ConfigConstant.appCfgPath + ap, JSON.toJSONString(cfg)); + } + }else{ + for (KeyValue keyValue : keyValues) { + String val = keyValue.getValue().toStringUtf8(); + AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); + appCfgMap.put(cfg.getApp(),cfg); + } + } + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java new file mode 100644 index 0000000..00998c8 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java @@ -0,0 +1,136 @@ +package com.jd.platform.hotkey.dashboard.common.monitor; + + +import cn.hutool.core.date.SystemClock; + +import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicInteger; + +/** + * 滑动窗口 + */ +public class SlidingWindow { + /** + * 循环队列，就是装多个窗口用，该数量是windowSize的2倍 + */ + private AtomicInteger[] timeSlices; + /** + * 队列的总长度 + */ + private int timeSliceSize; + /** + * 每个时间片的时长，以毫秒为单位 + */ + private int timeMillisPerSlice; + /** + * 共有多少个时间片（即窗口长度） + */ + private int windowSize; + /** + * 在一个完整窗口期内允许通过的最大阈值 + */ + private int threshold; + /** + * 该滑窗的起始创建时间，也就是第一个数据 + */ + private long beginTimestamp; + /** + * 最后一个数据的时间戳 + */ + private long lastAddTimestamp; + + public static void main(String[] args) { + //1秒一个时间片，窗口共5个 + SlidingWindow window = new SlidingWindow(10, 20); + + CyclicBarrier cyclicBarrier = new CyclicBarrier(10); + for (int i = 0; i < 10; i++) { + new Thread(() -> { + try { + cyclicBarrier.await(); + } catch (InterruptedException | BrokenBarrierException e) { + e.printStackTrace(); + } + int hot = window.addCount(2); + System.out.println(hot); + }).start(); + } + } + + + public SlidingWindow(int windowSize, int threshold) { + this.timeMillisPerSlice = 60 * 1000; + this.windowSize = windowSize; + this.threshold = threshold; + // 保证存储在至少两个window + this.timeSliceSize = windowSize * 2; + reset(); + } + + /** + * 初始化 + */ + private void reset() { + beginTimestamp = SystemClock.now(); + //窗口个数 + AtomicInteger[] localTimeSlices = new AtomicInteger[timeSliceSize]; + for (int i = 0; i < timeSliceSize; i++) { + localTimeSlices[i] = new AtomicInteger(0); + } + timeSlices = localTimeSlices; + } + + + private int locationIndex() { + long now = SystemClock.now(); + if (now - lastAddTimestamp > timeMillisPerSlice * windowSize) { reset(); } + int index = (int) (((now - beginTimestamp) / timeMillisPerSlice) % timeSliceSize); + return Math.max(index, 0); + } + + + public synchronized int addCount(int count) { + int index = locationIndex(); + clearFromIndex(index); + int sum = 0; + // 在当前时间片里继续+1 + sum += timeSlices[index].addAndGet(count); + //加上前面几个时间片 + for (int i = 1; i < windowSize; i++) { + sum += timeSlices[(index - i + timeSliceSize) % timeSliceSize].get(); + } + lastAddTimestamp = SystemClock.now(); + if(sum >= threshold){ + return sum; + } + return 0; + } + + private void clearFromIndex(int index) { + for (int i = 1; i <= windowSize; i++) { + int j = index + i; + if (j >= windowSize * 2) { + j -= windowSize * 2; + } + timeSlices[j].set(0); + } + } + + + public int getWindowSize() { + return windowSize; + } + + public void setWindowSize(int windowSize) { + this.windowSize = windowSize; + } + + public int getThreshold() { + return threshold; + } + + public void setThreshold(int threshold) { + this.threshold = threshold; + } +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/TestController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/TestController.java deleted file mode 100644 index 5d42394..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/TestController.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.jd.platform.hotkey.dashboard.controller; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-07-16 - */ -//@RestController -//public class TestController { -// @Resource -// private IConfigCenter configCenter; -// -// @RequestMapping("delete") -// public Object delete(String key) { -//// configCenter.putAndGrant(ConfigConstant.hotKeyRecordPath + "sample" + "/" + key, com.jd.platform.hotkey.common.tool.Constant.DEFAULT_DELETE_VALUE, 1); -// configCenter.putAndGrant(ConfigConstant.hotKeyRecordPath + "bbb" + "/" + key, "", 60); -//// configCenter.delete(ConfigConstant.hotKeyPath + "sample" + "/" + key); -// -// return "1"; -// } -//} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java index beabf4e..2bd7653 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java @@ -3,7 +3,7 @@ package com.jd.platform.hotkey.dashboard.erp; import com.jd.common.springmvc.interceptor.SpringSSOInterceptor; import com.jd.common.web.LoginContext; import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.service.UserService; +import com.jd.platform.hotkey.dashboard.biz.service.UserService; import org.springframework.beans.factory.annotation.Autowired; import javax.servlet.http.Cookie; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/EtcdConfig.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdConfig.java similarity index 93% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/EtcdConfig.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdConfig.java index 0f585b6..edb0708 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/EtcdConfig.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdConfig.java @@ -1,4 +1,4 @@ -package com.jd.platform.hotkey.dashboard.common.config; +package com.jd.platform.hotkey.dashboard.etcd; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.configcenter.etcd.JdEtcdBuilder; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java new file mode 100644 index 0000000..6714da9 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -0,0 +1,272 @@ +package com.jd.platform.hotkey.dashboard.etcd; + +import cn.hutool.core.util.StrUtil; +import com.ibm.etcd.api.Event; +import com.ibm.etcd.api.KeyValue; +import com.ibm.etcd.client.kv.KvClient; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.rule.KeyRule; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; +import com.jd.platform.hotkey.dashboard.common.domain.IRecord; +import com.jd.platform.hotkey.dashboard.common.monitor.DataHandler; +import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; +import com.jd.platform.hotkey.dashboard.common.monitor.PushHandler; +import com.jd.platform.hotkey.dashboard.model.Worker; +import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; +import com.jd.platform.hotkey.dashboard.biz.service.WorkerService; +import com.jd.platform.hotkey.dashboard.util.CommonUtil; +import com.jd.platform.hotkey.dashboard.util.RuleUtil; +import io.grpc.StatusRuntimeException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import javax.annotation.PostConstruct; +import javax.annotation.Resource; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * @ProjectName: hotkey + * @ClassName: EtcdMonitor + * @Author: liyunfeng31 + * @Date: 2020/4/18 18:29 + */ +@SuppressWarnings("ALL") +@Component +public class EtcdMonitor { + + private static Logger log = LoggerFactory.getLogger(EtcdMonitor.class); + + @Resource + private IConfigCenter configCenter; + + @Resource + private WorkerService workerService; + + @Resource + private SummaryMapper summaryMapper; + + @Resource + private DataHandler dataHandler; + + @Resource + private PushHandler pushHandler; + + + public static final ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(16); + + /** + * 监听新来的热key，该key的产生是来自于手工在控制台添加 + */ + public void watchHandOperationHotKey() { + threadPoolExecutor.submit(() -> { + KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyPath); + while (watchIterator.hasNext()) { + Event event = event(watchIterator); + KeyValue kv = event.getKv(); + String v = kv.getValue().toStringUtf8(); + Event.EventType eventType = event.getType(); + + String appKey = event.getKv().getKey().toStringUtf8().replace(ConfigConstant.hotKeyPath, ""); + dataHandler.offer(new IRecord() { + @Override + public String appNameKey() { + return appKey; + } + + @Override + public String value() { + return v; + } + + @Override + public int type() { + return eventType.getNumber(); + } + + @Override + public Date createTime() { + return new Date(); + } + }); + } + }); + } + + + @PostConstruct + public void startWatch() { + //拉取rules + fetchRuleFromEtcd(); + + //规则拉取完毕后才能去开始入库 + insertRecords(); + + //开始入库 + dealHotKey(); + + //监听手工创建的key + watchHandOperationHotKey(); + + //监听rule变化 + watchRule(); + +// watchWorkers(); + + //观察热key访问次数和总访问次数，并做统计 + watchHitCount(); + } + + /** + * 每秒去清理一次caffeine + */ + @Scheduled(fixedRate = 1000) + public void cleanCaffeine() { + try { + HotKeyReceiver.cleanUpCaffeine(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + private void insertRecords() { + threadPoolExecutor.submit(() -> { + dataHandler.insertRecords(); + }); + } + + + /** + * 开始消费各worker发来的热key + */ + private void dealHotKey() { + threadPoolExecutor.submit(() -> { + dataHandler.dealHotKey(); + }); + threadPoolExecutor.submit(() -> { + pushHandler.pushWarnMsg(); + }); + + } + + /** + * 异步监听rule规则变化 + */ + private void watchRule() { + threadPoolExecutor.submit(() -> { + + KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.rulePath); + //如果有新事件，即rule的变更，就重新拉取所有的信息 + while (watchIterator.hasNext()) { + //这句必须写，next会让他卡住，除非真的有新rule变更 + Event event = event(watchIterator); + + try { + log.info("---------watch rule change---------"+event.getType().toString()); + //全量拉取rule信息 + fetchRuleFromEtcd(); + } catch (Exception e) { + log.error("watch rule err"); + } + } + + }); + } + + /** + * 启动后从etcd拉取所有rule + */ + private void fetchRuleFromEtcd() { + RuleUtil.init(); + try { + List ruleList = new ArrayList<>(); + + //从etcd获取rule + List keyValues = configCenter.getPrefix(ConfigConstant.rulePath); + + for (KeyValue keyValue : keyValues) { + try { + log.info(keyValue.getKey() + "---" + keyValue.getValue()); + String appName = keyValue.getKey().toStringUtf8().replace(ConfigConstant.rulePath, ""); + if (StrUtil.isEmpty(appName)) { + configCenter.delete(keyValue.getKey().toStringUtf8()); + continue; + } + String rulesStr = keyValue.getValue().toStringUtf8(); + RuleUtil.put(appName, FastJsonUtils.toList(rulesStr, KeyRule.class)); + } catch (Exception e) { + log.error("rule parse failure"); + } + + } + + } catch (StatusRuntimeException ex) { + //etcd连不上 + log.error("etcd connected fail. Check the etcd address!!!"); + } + + } + + private void watchWorkers() { + threadPoolExecutor.submit(() -> { + KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.workersPath); + while (watchIterator.hasNext()) { + Event event = event(watchIterator); + KeyValue kv = event.getKv(); + Event.EventType eventType = event.getType(); + String k = kv.getKey().toStringUtf8(); + String v = kv.getValue().toStringUtf8(); + long version = kv.getModRevision(); + String uuid = k + Constant.JOIN + version; + Worker worker = new Worker(k, v, uuid); + if (eventType.equals(Event.EventType.PUT)) { + workerService.insertWorkerBySys(worker); + } else if (eventType.equals(Event.EventType.DELETE)) { + worker.setState(0); + workerService.updateWorker(worker); + } + } + }); + } + + + /** + * 监听热key访问次数和总访问次数 + */ + private void watchHitCount() { + threadPoolExecutor.submit(() -> { + KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.keyHitCountPath); + while (watchIterator.hasNext()) { + Event event = event(watchIterator); + KeyValue kv = event.getKv(); + Event.EventType eventType = event.getType(); + if (Event.EventType.DELETE.equals(eventType)) { + continue; + } + String k = kv.getKey().toStringUtf8(); + String v = kv.getValue().toStringUtf8(); + + try { + Map map = FastJsonUtils.stringToCollect(v); + for (String key : map.keySet()) { + int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map)); + } + } catch (Exception e) { + e.printStackTrace(); + } + + } + }); + } + + + private Event event(KvClient.WatchIterator watchIterator) { + return watchIterator.next().getEvents().get(0); + } + +} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java index 173a465..8fcd977 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java @@ -4,14 +4,12 @@ package com.jd.platform.hotkey.dashboard.interceptor; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; -import com.jd.platform.hotkey.dashboard.service.UserService; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import io.jsonwebtoken.Claims; import org.springframework.http.HttpMethod; import org.springframework.util.StringUtils; import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; -import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 21488ac..804bca3 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -54,10 +54,6 @@ public class CommonUtil { } - public static String keyName(String k) { - int index = k.lastIndexOf("/"); - return k.substring(index + 1); - } public static String encoder(String text) { @@ -81,49 +77,6 @@ public class CommonUtil { } - /** - * 拼装数据 - * - * @param list list-data - * @param startTime 开始时间 - * @param size 格子数 - * @param type 类型 1分钟 2小时 - * @return vo - */ - public static HotKeyLineChartVo assembleData(List list, LocalDateTime startTime, int size, int type) { - Set set = new TreeSet<>(); - boolean isHour = type == 1; - String suffix = isHour ? "60" : "24"; - String pattern = isHour ? DateUtil.PATTERN_MINUS : DateUtil.PATTERN_HOUR; - Map map = new HashMap<>(10); - Map> listMap = listGroup(list); -// log.info("按照rule分组以后的listMap--> {}", JSON.toJSONString(listMap)); - for (Map.Entry> m : listMap.entrySet()) { - int start = DateUtil.reviseTime(startTime, 0, type); - map.put(m.getKey(), new int[size]); - int[] data = map.get(m.getKey()); - int tmp = 0; - for (int i = 0; i < size; i++) { - if (String.valueOf(start).endsWith(suffix)) { - LocalDateTime tmpTime = DateUtil.strToLdt((start - 1) + "", pattern); - start = DateUtil.reviseTime(tmpTime, 1, type); - } -// log.info("start--> {}, tmp---> {} ", start, tmp); - set.add(DateUtil.strToLdt(start + "", pattern).toString().replace("T", " ")); - Statistics st = m.getValue().get(tmp); - int val = isHour ? st.getMinutes() : st.getHours(); - if (start != val) { - data[i] = 0; - } else { - tmp++; - data[i] = st.getCount(); - } - start++; - } - } - return new HotKeyLineChartVo(new ArrayList<>(set), map); - } - /** * 分组 @@ -132,9 +85,7 @@ public class CommonUtil { * @return map */ private static Map> listGroup(List list) { - if (Constant.VERSION != 1) { - return list.stream().collect(Collectors.groupingBy(Statistics::getRule)); - } + // return list.stream().collect(Collectors.groupingBy(Statistics::getRule)); return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); } diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index 1678e74..4fca4d2 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -27,18 +27,24 @@ spring : time-zone: GMT+8 date-format: yyyy-MM-dd HH:mm:ss datasource: - username: root - password: password - url: jdbc:mysql://localhost:3306/hotkey?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC + username: ${MYSQL_USER:root} + password: ${MYSQL_PASS:root} + url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/luck?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT%2B8 driver-class-name: com.mysql.cj.jdbc.Driver + hikari: + max-lifetime: 120000 + idle-timeout: 60000 + connection-timeout: 30000 + maximum-pool-size: 32 + minimum-idle: 10 pagehelper: helperDialect: mysql reasonable: true supportMethodsArguments: true params: count=countSql -mybatis: - configuration: - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl etcd: server: ${etcdServer:http://127.0.0.1:2379} diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml index 90b3b5d..ff382f1 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml @@ -1,6 +1,6 @@ - + @@ -16,7 +16,7 @@ id, biz_id, biz_type, from_str, to_str, app_name,update_user, create_time,uuid SELECT app,key_name,sum(count) as count - from hk_statistics - where 1=1 + from hk_statistics + where 1=1 and key_name LIKE CONCAT(CONCAT('%', #{key}), '%') @@ -53,7 +53,7 @@ = #{startTime} and create_time < #{endTime}]]> - GROUP BY key_name, app ORDER BY count desc + GROUP BY key_name, app ORDER BY count desc diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml index bd04545..34bb0d8 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml @@ -1,6 +1,6 @@ - + @@ -38,7 +38,7 @@ - select + select from hk_user where id = #{id,jdbcType=INTEGER} diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.xml index 5be300c..09e72b2 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.xml @@ -1,6 +1,6 @@ - + @@ -16,7 +16,7 @@ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+ + +
+
+ +
+
+
+ + + + + +
+ +
+ + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/appcfg/list.html b/dashboard/src/main/resources/templates/admin/appcfg/list.html new file mode 100644 index 0000000..19da89f --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/appcfg/list.html @@ -0,0 +1,166 @@ + + + + + + + + + + + + + + + +
+
+ +
+ +
+
+ +
+
+
搜索条件
+
+ +
+
+
+
+ + +
+ +
+
+
+
+ + + +
+ +
+ +
+
+
+ + + +
+ +
+ + + + + + + + + + + + + + diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index 668341d..3bd36d2 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -6,30 +6,30 @@

HotKeyv1

- - - -
-
-
-
-
-
+ + + +
+
+
+
+
+
- -
- +
+ diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index ee7b7fe..e663158 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -14,24 +14,24 @@
-
- - +
+ + - - + + - - + +
SELECT app,key_name,sum(count) as count - from hk_statistics - where 1=1 + from hk_statistics + where 1=1 and key_name LIKE CONCAT(CONCAT('%', #{key}), '%') @@ -53,7 +53,7 @@ = #{startTime} and create_time < #{endTime}]]> - GROUP BY key_name, app ORDER BY count desc + GROUP BY key_name, app ORDER BY count desc diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml index 34bb0d8..de74e50 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml @@ -38,7 +38,7 @@ -
-
-
- -
- -
-
-
- -
- -
-
-
- -
- -
-
-
- -
- -
-
-
-
- - -
-
- -
-
-
- - +
+
+
+
+
+
+
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+ +
+ +
+
+
+
+ + +
+
+
+
+
+
+
+
diff --git a/dashboard/src/main/resources/templates/admin/clear/edit.html b/dashboard/src/main/resources/templates/admin/clear/edit.html index 4ced25c..dd2b4d8 100644 --- a/dashboard/src/main/resources/templates/admin/clear/edit.html +++ b/dashboard/src/main/resources/templates/admin/clear/edit.html @@ -1,55 +1,55 @@ + > - + - - - - - - + + + + + +
-
-
-
-
-
-
-
-
-
- -
- -
-
-
- -
- -
-
+
+
+
+
+
+
+
+ +
+ +
+ +
+
+
+ +
+ +
+
-
-
- - -
-
- -
-
-
-
-
+
+
+ + +
+
+ +
+
+
+
+
diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index 3bd36d2..4764768 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -6,30 +6,30 @@

HotKeyv1

- - - -
-
-
-
-
-
+ + + +
+
+
+
+
+
- -
- +
+ diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index e663158..75e714a 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -14,24 +14,24 @@
-
- - +
+ + - - + + - - + +
+ + + and token = #{token} + and flag = #{flag} + and CREATED_BY = #{createdBy} + and CREATED_TIME = #{createdTime} + and UPDATED_BY = #{updatedBy} + and UPDATED_TIME = #{updatedTime} + + + + + + + insert into biz_access_token + + token, + flag, + CREATED_BY, + CREATED_TIME, + UPDATED_BY, + UPDATED_TIME, + + + #{token}, + #{flag}, + #{createdBy}, + #{createdTime}, + #{updatedBy}, + #{updatedTime}, + + + + + update biz_access_token + + token = #{token}, + flag = #{flag}, + CREATED_BY = #{createdBy}, + CREATED_TIME = #{createdTime}, + UPDATED_BY = #{updatedBy}, + UPDATED_TIME = #{updatedTime}, + + where id = #{id} + + + + delete from biz_access_token where id = #{id} + + \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.xml similarity index 100% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/ChangeLogMapper.xml diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml similarity index 100% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.xml similarity index 100% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/RulesMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/RulesMapper.xml diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.xml similarity index 100% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/StatisticsMapper.xml diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.xml similarity index 100% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.xml diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.xml similarity index 100% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.xml diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml similarity index 100% rename from dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/WorkerMapper.xml rename to dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/WorkerMapper.xml diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index 98fbd82..7497750 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -1,3 +1,8 @@ +-- !!! 注意设置sql model 否则可能sql报错 ！！！ +-- 查询你的sql_model参数：select @@global.sql_mode; 发现ONLY_FULL_GROUP_BY 则会导致报错 +-- 解决方式：set @@global.sql_mode='STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION' +-- 详情查阅：https://www.cnblogs.com/hjhsblogs/p/11079356.html + DROP TABLE IF EXISTS `hk_change_log`; CREATE TABLE `hk_change_log` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', @@ -120,3 +125,19 @@ CREATE TABLE `hk_summary` ( ) ENGINE = InnoDB AUTO_INCREMENT = 18 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin COMMENT = '汇总表' ROW_FORMAT = Compact; + +/* 请确认以下SQL符合您的变更需求，务必确认无误后再提交执行 */ + +CREATE TABLE `biz_access_token` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', + `token` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT 'token', + `flag` varchar(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL DEFAULT '' COMMENT 'flag', + `CREATED_BY` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '创建人', + `CREATED_TIME` datetime NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '创建时间', + `UPDATED_BY` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NULL COMMENT '修改人', + `UPDATED_TIME` datetime NULL ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间', + PRIMARY KEY (`id`) +) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin COMMENT = '咚咚token表' ROW_FORMAT = Compact; + + + diff --git a/dashboard/src/main/resources/templates/admin/appcfg/edit.html b/dashboard/src/main/resources/templates/admin/appcfg/edit.html index f628f38..d636f29 100644 --- a/dashboard/src/main/resources/templates/admin/appcfg/edit.html +++ b/dashboard/src/main/resources/templates/admin/appcfg/edit.html @@ -38,15 +38,21 @@
- +
- +
- + +
+
+
+ +
+
diff --git a/dashboard/src/main/resources/templates/admin/appcfg/list.html b/dashboard/src/main/resources/templates/admin/appcfg/list.html index 19da89f..c26b982 100644 --- a/dashboard/src/main/resources/templates/admin/appcfg/list.html +++ b/dashboard/src/main/resources/templates/admin/appcfg/list.html @@ -103,7 +103,8 @@ { field: 'app', title: 'app',width:200,align:"center"}, { field: 'dataTtl', title: '数据保存时长(天)',align:"center"}, { field: 'warnPeriod', title: '警报周期(秒)',align:"center"}, - { field: 'warnThreshold', title: '警报阈值',align:"center"}, + { field: 'warnMax', title: '警报最大阈值',align:"center"}, + { field: 'warnMin', title: '警报最小阈值',align:"center"}, { field: 'modifier', title: '修改人',align:"center"}, { field: 'warn', title: '是否开启警报', formatter: function (value) { -- Gitee From a68ec8ca325f0d630ba56a9e9481ca836a05525f Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 21 Oct 2020 17:35:39 +0800 Subject: [PATCH 25/59] merge erp --- dashboard/src/main/resources/application.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index f01cbc0..c190632 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -27,9 +27,9 @@ spring : time-zone: GMT+8 date-format: yyyy-MM-dd HH:mm:ss datasource: - username: zxc - password: zxc4851225 - url: jdbc:mysql://rm-bp1f55o24g3im17w8yo.mysql.rds.aliyuncs.com:3306/luck?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT + username: ${MYSQL_USER:root} + password: ${MYSQL_PASS:root} + url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/luck?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT driver-class-name: com.mysql.cj.jdbc.Driver hikari: max-lifetime: 120000 -- Gitee From ea55843ef7c86dfa9d09f53d31b0bf49a2098096 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 22 Oct 2020 16:20:03 +0800 Subject: [PATCH 26/59] =?UTF-8?q?=E5=92=9A=E5=92=9A=E6=8A=A5=E8=AD=A6?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=89=AB=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/biz/mapper/KeyRecordMapper.java | 3 + .../dashboard/biz/mapper/UserMapper.java | 2 + .../biz/service/impl/AppCfgServiceImpl.java | 1 - .../dashboard/common/domain/Constant.java | 6 + .../dashboard/common/domain/vo/AppCfgVo.java | 8 +- .../dashboard/common/monitor/DataHandler.java | 71 +++++++++++- .../dashboard/common/monitor/PushHandler.java | 104 +++--------------- .../hotkey/dashboard/etcd/EtcdMonitor.java | 4 - .../dashboard/warn/DongDongApiManager.java | 4 +- .../hotkey/dashboard/warn/DongDongUtil.java | 4 +- .../dashboard/biz/mapper/KeyRecordMapper.xml | 14 +++ .../dashboard/biz/mapper/UserMapper.xml | 7 ++ 12 files changed, 122 insertions(+), 106 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java index dde2c25..e8714aa 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.java @@ -27,4 +27,7 @@ public interface KeyRecordMapper { List statisticsByRule(SearchReq req); int clearExpireData(String app, Date expireDate); + + int countKeyRecord(SearchReq req); + } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java index 257a27d..30f0be8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.java @@ -25,4 +25,6 @@ public interface UserMapper { List selectHkUserList(User user); User selectByUserName(String userName); + + List listErpByApp(String app); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index 82ba42a..ba7162a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -76,7 +76,6 @@ public class AppCfgServiceImpl implements AppCfgService { @Override public void saveAppCfgVo(AppCfgVo cfg) { cfg.setWindow(new SlidingWindow(cfg.getWarnPeriod()/60,cfg.getWarnMin(),cfg.getWarnMax())); - PushHandler.appCfgMap.put(cfg.getApp(), cfg); configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java index d47ca2e..30dca53 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java @@ -66,4 +66,10 @@ public class Constant { static { HEAD.add("热点key"); HEAD.add("次数"); HEAD.add("所属APP");} + + public static final int WARN_INIT_MIN = -1; + + + public static final int WARN_INIT_MAX = 5000; + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java index 53b2cc0..0e7b2e8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; import java.io.Serializable; @@ -130,13 +131,12 @@ public class AppCfgVo implements Serializable { public AppCfgVo(String app) { this.app = app; this.dataTtl = 30; - this.warnPeriod = 10*60; - this.warnMin = -1; - this.warnMax = 1000; + this.warnPeriod = 60; + this.warnMin = Constant.WARN_INIT_MIN; + this.warnMax = Constant.WARN_INIT_MAX; this.version = 0L; this.warn = 1; this.modifier = "SYSTEM"; - this.window = new SlidingWindow(warnPeriod/60,warnMin,warnMax); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 8216cdd..379352c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -26,6 +26,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.Resource; +import javax.crypto.Mac; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Date; @@ -241,6 +242,27 @@ public class DataHandler { } + /** + * 每10秒检测一次热点记录 用于监控报警 + */ + @Scheduled(initialDelay = 30000,fixedRate = 10000) + public void scanRecordForMonitor() { + try { + SearchReq req = new SearchReq(); + Date date = new Date(); + req.setEndTime(date); + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + for (KeyValue kv : keyValues) { + queryRecordAndCheck( kv, req, date); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + + + + public void dealHotKey() { while (true) { try { @@ -269,8 +291,53 @@ public class DataHandler { @Override public Date createTime() { return new Date(); } }); - // 监控和推送 - pushHandler.monitorAndPush(app); + } + + + /** + * 查询条数 比对配置 发送报警 + */ + private void queryRecordAndCheck(KeyValue kv, SearchReq req, Date date){ + String val = kv.getValue().toStringUtf8(); + AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); + appendCfg(cfg); + if(cfg.getWarn() != 1){ return; } + + req.setApp(cfg.getApp()); + req.setStartTime(new Date(date.getTime() - cfg.getWarnPeriod()*1000)); + + int count = keyRecordMapper.countKeyRecord(req); + int type = 0; + if(count > cfg.getWarnMax()){ + type = 1; + }else if(count < cfg.getWarnMax()){ + type = 2; + } + + if(type > 0){ + String str = type == 1 ? "高于最大":"低于最小"; + int threshold = type == 1 ? cfg.getWarnMax():cfg.getWarnMin(); + String content = String.format("应用：%s热点记录在%d秒内累计: %d, %s阈值: %d, 请注意", cfg.getApp(),cfg.getWarnPeriod(),count,str,threshold); + pushHandler.pushMsg(cfg.getApp(),date,content); + } + } + + + + private void appendCfg(AppCfgVo cfg){ + + boolean maxNull = cfg.getWarnMax() == null; + boolean minNull = cfg.getWarnMin() == null; + + if(maxNull){ + cfg.setWarnMax(Constant.WARN_INIT_MAX); + } + if(minNull){ + cfg.setWarnMin(Constant.WARN_INIT_MIN); + } + if(maxNull || minNull){ + configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); + } } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java index 70a70e0..dd6ce94 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java @@ -1,26 +1,17 @@ package com.jd.platform.hotkey.dashboard.common.monitor; -import com.alibaba.fastjson.JSON; -import com.ibm.etcd.api.KeyValue; -import com.jd.platform.hotkey.common.configcenter.ConfigConstant; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.dashboard.biz.service.UserService; -import com.jd.platform.hotkey.dashboard.common.domain.PushMsgWrapper; -import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; + +import com.jd.platform.hotkey.dashboard.biz.mapper.UserMapper; import com.jd.platform.hotkey.dashboard.warn.DongDongApiManager; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; - import javax.annotation.Resource; -import java.util.HashMap; +import java.util.Date; import java.util.List; import java.util.Map; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.LinkedBlockingQueue; /** * @ProjectName: hotkey @@ -46,69 +37,29 @@ public class PushHandler { */ private static final long INTERVAL = 10*60*1000L; - /** - * app-config map - */ - public static Map appCfgMap = new ConcurrentHashMap<>(); /** * app-time 用于app存储报警时间 做拦截 */ private static Map appIntervalMap = new ConcurrentHashMap<>(); - /** - * 超过阈值的热点记录次数 放到队列里 - */ - private static final BlockingQueue MSG_QUEUE = new LinkedBlockingQueue<>(); - - - @Resource - private IConfigCenter configCenter; - - @Resource - private UserService userService; @Resource private DongDongApiManager apiManager; - /** - * 监控和入队 - */ - public void monitorAndPush(String app) throws InterruptedException { - AppCfgVo cfg = PushHandler.appCfgMap.get(app); - if(cfg.getWarn().equals(1)){ - SlidingWindow wd = cfg.getWindow(); - int count = wd.addCount(1); - if(count == 0){ return; } - MSG_QUEUE.put(new PushMsgWrapper(app, count)); - } - } + @Resource + private UserMapper userMapper; /** - * 启动线程处理警报消息队列 + * 发送消息 */ - public void pushWarnMsg() { - // 初始化MAP到内存 - initAppCfgMap(); - logger.info("initAppCfgMap success AppCfgMap:{}",JSON.toJSONString(appCfgMap)); - - while (true){ - try { - PushMsgWrapper msgWrapper = MSG_QUEUE.take(); - String warnApp = msgWrapper.getApp(); - Long msgTime = msgWrapper.getDate(); - boolean send = check(warnApp, msgTime); - if(send){ - Integer ct = msgWrapper.getCount(); - String content = ct == -1 ? warnApp+"热点记录频率低于最小阈值，请注意！" : warnApp+"热点记录频率高于最大阈值，请注意！"; - apiManager.push(TITLE,content); - }else{ - logger.info("check fail不允许发送：msg: {}",JSON.toJSONString(msgWrapper)); - } - } catch (InterruptedException e) { - logger.info("pushWarnMsg thread error , msg :{}", e.getMessage()); - } + public void pushMsg(String app, Date msgTime, String content) { + boolean send = check(app, msgTime.getTime()); + if(send){ + logger.info("Warn PushMsg content:{}",content); + List erpList = userMapper.listErpByApp(app); + apiManager.push(TITLE,content,erpList); } } @@ -133,33 +84,4 @@ public class PushHandler { return false; } - - /** - * 初始化cfgMap和滑动窗口 - */ - private void initAppCfgMap() { - /** - * 为了加入最小值 - */ - configCenter.delete(ConfigConstant.appCfgPath); - List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); - if(CollectionUtils.isEmpty(keyValues) || keyValues.size()==1){ - List apps = userService.listApp(); - for (String ap : apps) { - AppCfgVo cfg = new AppCfgVo(ap); - appCfgMap.put(ap,cfg); - configCenter.put(ConfigConstant.appCfgPath + ap, JSON.toJSONString(cfg)); - } - return; - } - - for (KeyValue keyValue : keyValues) { - String val = keyValue.getValue().toStringUtf8(); - if(StringUtils.isNotEmpty(val)){ - AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); - appCfgMap.put(cfg.getApp(),cfg); - } - } - } - } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java index 6714da9..2f3bc11 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -148,10 +148,6 @@ public class EtcdMonitor { threadPoolExecutor.submit(() -> { dataHandler.dealHotKey(); }); - threadPoolExecutor.submit(() -> { - pushHandler.pushWarnMsg(); - }); - } /** diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java index 88b83e7..25d2dd2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java @@ -74,7 +74,7 @@ public class DongDongApiManager { } - public boolean push(String title, String content) { + public boolean push(String title, String content, List erpList) { List tokens = accessTokenService.selectBizAccessTokenList(new BizAccessToken()); if(tokens.size() == 0){ log.error("没有查询到有效的token！"); @@ -85,7 +85,7 @@ public class DongDongApiManager { return false; } try { - return dongUtil.push(title,content,accessToken); + return dongUtil.push(title,content,accessToken, erpList); }catch (Exception e){ return false; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java index 626010c..364fe3a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java @@ -71,7 +71,7 @@ public class DongDongUtil { } - public boolean push(String title,String content,String accessToken) throws Exception{ + public boolean push(String title,String content,String accessToken, List erpList) throws Exception{ String dogUrl = "http://dog-ee.jd.com/push"; List params = new ArrayList<>(); params.add(new BasicNameValuePair("accessid", UUID.randomUUID().toString())); @@ -94,7 +94,7 @@ public class DongDongUtil { List pins = new ArrayList<>(); pins.add("wuweifeng10"); pins.add("liyunfeng31"); - pins.add("erp"); + pins.addAll(erpList); json.put("tos", pins); JSONObject extend = new JSONObject(); diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml index 5f347bf..82274d2 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml @@ -139,6 +139,20 @@ + + + + INSERT ignore INTO hk_key_record( diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.xml index 5ce765d..afe87f1 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/UserMapper.xml @@ -170,4 +170,11 @@ group by app_name + + + \ No newline at end of file -- Gitee From 186677a15e6d40b6f8d5073ee7b2a5e2b864f8c1 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 22 Oct 2020 16:23:10 +0800 Subject: [PATCH 27/59] =?UTF-8?q?=E5=92=9A=E5=92=9A=E6=8A=A5=E8=AD=A6?= =?UTF-8?q?=E6=94=B9=E4=B8=BA=E6=89=AB=E5=BA=93=20=E5=88=9D=E5=A7=8B?= =?UTF-8?q?=E5=80=BC=E5=8F=98=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/common/monitor/DataHandler.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 379352c..25f94ab 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -335,9 +335,8 @@ public class DataHandler { if(minNull){ cfg.setWarnMin(Constant.WARN_INIT_MIN); } - if(maxNull || minNull){ - configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); - } + cfg.setWarnPeriod(60); + configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); } } -- Gitee From 1f1df77dd68ee11ea36765e9121c2f601559c9bb Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 22 Oct 2020 16:32:17 +0800 Subject: [PATCH 28/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=BB=91=E5=8A=A8?= =?UTF-8?q?=E7=AA=97=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/biz/service/impl/AppCfgServiceImpl.java | 1 - .../hotkey/dashboard/common/domain/vo/AppCfgVo.java | 11 ----------- 2 files changed, 12 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index ba7162a..c49fc8c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -75,7 +75,6 @@ public class AppCfgServiceImpl implements AppCfgService { */ @Override public void saveAppCfgVo(AppCfgVo cfg) { - cfg.setWindow(new SlidingWindow(cfg.getWarnPeriod()/60,cfg.getWarnMin(),cfg.getWarnMax())); configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java index 0e7b2e8..9975e18 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java @@ -50,9 +50,6 @@ public class AppCfgVo implements Serializable { */ private String modifier; - - private SlidingWindow window; - public String getApp() { return app; } @@ -117,14 +114,6 @@ public class AppCfgVo implements Serializable { this.modifier = modifier; } - public SlidingWindow getWindow() { - return window; - } - - public void setWindow(SlidingWindow window) { - this.window = window; - } - public AppCfgVo() { } -- Gitee From 4994680401e0320efb43c62c286cd711d1e9fd40 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 22 Oct 2020 16:34:28 +0800 Subject: [PATCH 29/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=BB=91=E5=8A=A8?= =?UTF-8?q?=E7=AA=97=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/service/impl/AppCfgServiceImpl.java | 2 - .../dashboard/common/domain/vo/AppCfgVo.java | 2 +- .../common/monitor/SlidingWindow.java | 135 ------------------ 3 files changed, 1 insertion(+), 138 deletions(-) delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index c49fc8c..04f3a30 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -9,8 +9,6 @@ import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; import com.jd.platform.hotkey.dashboard.biz.service.AppCfgService; -import com.jd.platform.hotkey.dashboard.common.monitor.PushHandler; -import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; import com.jd.platform.hotkey.dashboard.util.PageUtil; import org.springframework.stereotype.Service; import org.springframework.util.StringUtils; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java index 9975e18..4809303 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/AppCfgVo.java @@ -2,7 +2,7 @@ package com.jd.platform.hotkey.dashboard.common.domain.vo; import com.jd.platform.hotkey.dashboard.common.domain.Constant; -import com.jd.platform.hotkey.dashboard.common.monitor.SlidingWindow; + import java.io.Serializable; /** diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java deleted file mode 100644 index 9dcbcec..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/SlidingWindow.java +++ /dev/null @@ -1,135 +0,0 @@ -package com.jd.platform.hotkey.dashboard.common.monitor; - - -import cn.hutool.core.date.SystemClock; - -import java.util.concurrent.BrokenBarrierException; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * 滑动窗口 - */ -public class SlidingWindow { - /** - * 循环队列，就是装多个窗口用，该数量是windowSize的2倍 - */ - private AtomicInteger[] timeSlices; - /** - * 队列的总长度 - */ - private int timeSliceSize; - /** - * 每个时间片的时长，以毫秒为单位 - */ - private int timeMillisPerSlice; - /** - * 共有多少个时间片（即窗口长度） - */ - private int windowSize; - /** - * 在一个完整窗口期内允许通过的最大阈值 - */ - private int max; - - /** - * 在一个完整窗口期内最小阈值 - */ - private int min; - /** - * 该滑窗的起始创建时间，也就是第一个数据 - */ - private long beginTimestamp; - /** - * 最后一个数据的时间戳 - */ - private long lastAddTimestamp; - - - public SlidingWindow(int windowSize, int min, int max) { - this.timeMillisPerSlice = 60 * 1000; - this.windowSize = windowSize; - this.max = max; - this.min = min; - // 保证存储在至少两个window - this.timeSliceSize = windowSize * 2; - reset(); - } - - /** - * 初始化 - */ - private void reset() { - beginTimestamp = SystemClock.now(); - //窗口个数 - AtomicInteger[] localTimeSlices = new AtomicInteger[timeSliceSize]; - for (int i = 0; i < timeSliceSize; i++) { - localTimeSlices[i] = new AtomicInteger(0); - } - timeSlices = localTimeSlices; - } - - - private int locationIndex() { - long now = SystemClock.now(); - if (now - lastAddTimestamp > timeMillisPerSlice * windowSize) { reset(); } - int index = (int) (((now - beginTimestamp) / timeMillisPerSlice) % timeSliceSize); - return Math.max(index, 0); - } - - - public synchronized int addCount(int count) { - int index = locationIndex(); - clearFromIndex(index); - int sum = 0; - // 在当前时间片里继续+1 - sum += timeSlices[index].addAndGet(count); - //加上前面几个时间片 - for (int i = 1; i < windowSize; i++) { - sum += timeSlices[(index - i + timeSliceSize) % timeSliceSize].get(); - } - lastAddTimestamp = SystemClock.now(); - if(sum >= max){ - return sum; - }else if(sum < min){ - return -1; - }else{ - return 0; - } - } - - private void clearFromIndex(int index) { - for (int i = 1; i <= windowSize; i++) { - int j = index + i; - if (j >= windowSize * 2) { - j -= windowSize * 2; - } - timeSlices[j].set(0); - } - } - - - public int getWindowSize() { - return windowSize; - } - - public void setWindowSize(int windowSize) { - this.windowSize = windowSize; - } - - public int getMax() { - return max; - } - - public void setMax(int max) { - this.max = max; - } - - public int getMin() { - return min; - } - - public void setMin(int min) { - this.min = min; - } -} -- Gitee From 28d79e6850b783195a77032f5a64c4faff7a3844 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 22 Oct 2020 16:51:09 +0800 Subject: [PATCH 30/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=BB=91=E5=8A=A8?= =?UTF-8?q?=E7=AA=97=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/common/monitor/DataHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 25f94ab..9a6d59a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -310,7 +310,7 @@ public class DataHandler { int type = 0; if(count > cfg.getWarnMax()){ type = 1; - }else if(count < cfg.getWarnMax()){ + }else if(count < cfg.getWarnMin()){ type = 2; } -- Gitee From 45f485d3a616e0924c6adc778c1afb166e865b2c Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 22 Oct 2020 16:59:56 +0800 Subject: [PATCH 31/59] merge erp --- .../worker/tool/DirectMemoryReporter.java | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/tool/DirectMemoryReporter.java diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/DirectMemoryReporter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/DirectMemoryReporter.java new file mode 100644 index 0000000..689ed83 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/DirectMemoryReporter.java @@ -0,0 +1,40 @@ +package com.jd.platform.hotkey.worker.tool; + +import io.netty.util.internal.PlatformDependent; +import org.springframework.stereotype.Component; +import org.springframework.util.ReflectionUtils; + +import javax.annotation.PostConstruct; +import java.lang.reflect.Field; +import java.util.concurrent.atomic.AtomicLong; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-10-22 + */ +@Component +public class DirectMemoryReporter { + private static final String BUSINESS_KEY = "netty_direct_memory"; + private AtomicLong directMemory; + + @PostConstruct + public void init(){ + //通过反射拿到netty进行堆外内存统计字段 + Field field = ReflectionUtils.findField(PlatformDependent.class,"DIRECT_MEMORY_COUNTER"); + field.setAccessible(true); + try{ + directMemory = ((AtomicLong)field.get(PlatformDependent.class)); + }catch (Exception e){ + + } + } + + public void doReport(String processName){ + try{ + long memoryInb = directMemory.get(); + }catch (Exception e){ + + } + } +} \ No newline at end of file -- Gitee From 2ec60d88ee4d018396a0c302c2bb047230c84578 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 22 Oct 2020 17:36:04 +0800 Subject: [PATCH 32/59] =?UTF-8?q?=E9=98=88=E5=80=BC=E5=BC=80=E9=97=AD?= =?UTF-8?q?=E5=8C=BA=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/common/monitor/DataHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 9a6d59a..6374ba4 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -308,9 +308,9 @@ public class DataHandler { int count = keyRecordMapper.countKeyRecord(req); int type = 0; - if(count > cfg.getWarnMax()){ + if(count >= cfg.getWarnMax()){ type = 1; - }else if(count < cfg.getWarnMin()){ + }else if(count <= cfg.getWarnMin()){ type = 2; } -- Gitee From 0bcaeaf9fde1d5f0c46cdee03883c220628b4f71 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 22 Oct 2020 17:38:01 +0800 Subject: [PATCH 33/59] =?UTF-8?q?=E9=98=88=E5=80=BC=E5=BC=80=E9=97=AD?= =?UTF-8?q?=E5=8C=BA=E9=97=B4=E4=BF=AE=E6=94=B9-=E6=8A=A5=E8=AD=A6?= =?UTF-8?q?=E9=99=90=E5=88=B6=E9=97=B4=E9=9A=94=E4=BB=8E10-2=E5=88=86?= =?UTF-8?q?=E9=92=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/common/monitor/PushHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java index dd6ce94..1b926de 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java @@ -33,9 +33,9 @@ public class PushHandler { private static final String TITLE = "hotKey异常提醒"; /** - * 拦截重复报警间隔 10分钟 + * 拦截重复报警间隔 2分钟 */ - private static final long INTERVAL = 10*60*1000L; + private static final long INTERVAL = 2*60*1000L; /** -- Gitee From 85fa1df2c69a716e2e848f5c7fb92ff373cddd95 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 27 Oct 2020 16:35:35 +0800 Subject: [PATCH 34/59] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/common/monitor/DataHandler.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 6374ba4..b3c954b 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -92,10 +92,11 @@ public class DataHandler { keyRecordList.add(keyRecord); } } - + log.info("batch insert record size:{}",keyRecordList.size()); keyRecordMapper.batchInsert(keyRecordList); - } catch (InterruptedException e) { + } catch (Exception e) { + log.error("batch insert error:{}",e.getMessage(), e); e.printStackTrace(); } } @@ -300,13 +301,14 @@ public class DataHandler { private void queryRecordAndCheck(KeyValue kv, SearchReq req, Date date){ String val = kv.getValue().toStringUtf8(); AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); - appendCfg(cfg); + // appendCfg(cfg); if(cfg.getWarn() != 1){ return; } req.setApp(cfg.getApp()); req.setStartTime(new Date(date.getTime() - cfg.getWarnPeriod()*1000)); int count = keyRecordMapper.countKeyRecord(req); + log.info("应用app:{}, 记录count:{}, 统计时间Period:{}",cfg.getApp(), count, cfg.getWarnPeriod()); int type = 0; if(count >= cfg.getWarnMax()){ type = 1; @@ -319,6 +321,8 @@ public class DataHandler { int threshold = type == 1 ? cfg.getWarnMax():cfg.getWarnMin(); String content = String.format("应用：%s热点记录在%d秒内累计: %d, %s阈值: %d, 请注意", cfg.getApp(),cfg.getWarnPeriod(),count,str,threshold); pushHandler.pushMsg(cfg.getApp(),date,content); + }else{ + log.error(" app config error out of rang: {}",val); } } -- Gitee From 620d25e528624e1dedf128ff65ab7783e4cf214a Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 27 Oct 2020 16:43:16 +0800 Subject: [PATCH 35/59] merge erp --- .../dashboard/common/monitor/DataHandler.java | 81 +++++++++---------- 1 file changed, 36 insertions(+), 45 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index b3c954b..3f7c617 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -26,7 +26,6 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.Resource; -import javax.crypto.Mac; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Date; @@ -92,11 +91,10 @@ public class DataHandler { keyRecordList.add(keyRecord); } } - log.info("batch insert record size:{}",keyRecordList.size()); keyRecordMapper.batchInsert(keyRecordList); } catch (Exception e) { - log.error("batch insert error:{}",e.getMessage(), e); + log.error("batch insert error:{}", e.getMessage(), e); e.printStackTrace(); } } @@ -218,7 +216,6 @@ public class DataHandler { } - /** * 每天根据app的配置清理过期数据 */ @@ -236,7 +233,7 @@ public class DataHandler { keyRecordMapper.clearExpireData(app, expireDate); statisticsMapper.clearExpireData(app, expireDate); } - } catch (Exception e) { + } catch (Exception e) { e.printStackTrace(); } @@ -246,7 +243,7 @@ public class DataHandler { /** * 每10秒检测一次热点记录 用于监控报警 */ - @Scheduled(initialDelay = 30000,fixedRate = 10000) + @Scheduled(initialDelay = 30000, fixedRate = 10000) public void scanRecordForMonitor() { try { SearchReq req = new SearchReq(); @@ -254,7 +251,7 @@ public class DataHandler { req.setEndTime(date); List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); for (KeyValue kv : keyValues) { - queryRecordAndCheck( kv, req, date); + queryRecordAndCheck(kv, req, date); } } catch (Exception e) { e.printStackTrace(); @@ -262,15 +259,13 @@ public class DataHandler { } - - public void dealHotKey() { while (true) { try { //获取发来的这个热key，存入本地caffeine，设置过期时间 HotKeyModel model = HotKeyReceiver.take(); //将该key放入实时热key本地缓存中 - if(model != null){ + if (model != null) { HotKeyReceiver.put(model); putRecord(model.getAppName(), model.getKey()); } @@ -281,16 +276,27 @@ public class DataHandler { } - public void putRecord(String app, String key) throws InterruptedException { + private void putRecord(String app, String key) { this.offer(new IRecord() { @Override - public String appNameKey() { return app + "/" + key; } + public String appNameKey() { + return app + "/" + key; + } + @Override - public String value() { return UUID.randomUUID().toString(); } + public String value() { + return UUID.randomUUID().toString(); + } + @Override - public int type() { return 0; } + public int type() { + return 0; + } + @Override - public Date createTime() { return new Date(); } + public Date createTime() { + return new Date(); + } }); } @@ -298,49 +304,34 @@ public class DataHandler { /** * 查询条数 比对配置 发送报警 */ - private void queryRecordAndCheck(KeyValue kv, SearchReq req, Date date){ + private void queryRecordAndCheck(KeyValue kv, SearchReq req, Date date) { String val = kv.getValue().toStringUtf8(); AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); - // appendCfg(cfg); - if(cfg.getWarn() != 1){ return; } + if (cfg.getWarn() != 1) { + return; + } req.setApp(cfg.getApp()); - req.setStartTime(new Date(date.getTime() - cfg.getWarnPeriod()*1000)); + req.setStartTime(new Date(date.getTime() - cfg.getWarnPeriod() * 1000)); int count = keyRecordMapper.countKeyRecord(req); - log.info("应用app:{}, 记录count:{}, 统计时间Period:{}",cfg.getApp(), count, cfg.getWarnPeriod()); + log.info("应用app:{}, 记录count:{}, 统计时间Period:{}", cfg.getApp(), count, cfg.getWarnPeriod()); int type = 0; - if(count >= cfg.getWarnMax()){ + if (count >= cfg.getWarnMax()) { type = 1; - }else if(count <= cfg.getWarnMin()){ + } else if (count <= cfg.getWarnMin()) { type = 2; } - if(type > 0){ - String str = type == 1 ? "高于最大":"低于最小"; - int threshold = type == 1 ? cfg.getWarnMax():cfg.getWarnMin(); - String content = String.format("应用：%s热点记录在%d秒内累计: %d, %s阈值: %d, 请注意", cfg.getApp(),cfg.getWarnPeriod(),count,str,threshold); - pushHandler.pushMsg(cfg.getApp(),date,content); - }else{ - log.error(" app config error out of rang: {}",val); + if (type > 0) { + String str = type == 1 ? "高于最大" : "低于最小"; + int threshold = type == 1 ? cfg.getWarnMax() : cfg.getWarnMin(); + String content = String.format("应用：%s热点记录在%d秒内累计: %d, %s阈值: %d, 请注意", cfg.getApp(), cfg.getWarnPeriod(), count, str, threshold); + pushHandler.pushMsg(cfg.getApp(), date, content); + } else { + log.error(" app config error out of rang: {}", val); } } - - private void appendCfg(AppCfgVo cfg){ - - boolean maxNull = cfg.getWarnMax() == null; - boolean minNull = cfg.getWarnMin() == null; - - if(maxNull){ - cfg.setWarnMax(Constant.WARN_INIT_MAX); - } - if(minNull){ - cfg.setWarnMin(Constant.WARN_INIT_MIN); - } - cfg.setWarnPeriod(60); - configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); - } - } -- Gitee From b0a6172b91c2c69c94583dfa9628e172be03699e Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 28 Oct 2020 18:52:35 +0800 Subject: [PATCH 36/59] merge erp --- dashboard/src/main/resources/application.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index c190632..66bb0d7 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -50,9 +50,9 @@ etcd: server: ${etcdServer:http://127.0.0.1:2379} erp: enabled: false - ssoAppKey: fangzhou - ssoAppToken: 3f48d7ade61940f2ba22079ebc273fb8 + ssoAppKey: xxx + ssoAppToken: xxx loginUrl: https://ssa.jd.com/sso/login excludePath: /login,/assets,/bootstarp,/common,/htmlformat,/images,/jn,/jquery,/user/login,login ssoAppUrl: http://ssa.jd.com - defaultPwd: 123456 + defaultPwd: xxx -- Gitee From 410e74b8c2971fa2571aaff8336de9a88f028160 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 29 Oct 2020 15:04:24 +0800 Subject: [PATCH 37/59] =?UTF-8?q?=E5=BA=94=E7=94=A8=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E7=9A=84=E6=97=B6=E5=80=99=20=E9=85=8D=E7=BD=AE=E8=81=94?= =?UTF-8?q?=E5=8A=A8=E4=BF=AE=E6=94=B9=20=20=E4=BB=A5=E5=8F=8A=E6=89=BE?= =?UTF-8?q?=E5=9B=9E=20=E8=A7=84=E5=88=99=E5=8F=98=E6=9B=B4=E8=AE=B0?= =?UTF-8?q?=E5=BD=95=E4=BF=9D=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/service/impl/AppCfgServiceImpl.java | 18 +++++++++++++++--- .../biz/service/impl/RuleServiceImpl.java | 7 ++++++- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index 04f3a30..ebbc80a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -5,6 +5,7 @@ import com.github.pagehelper.util.StringUtil; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.dashboard.biz.mapper.UserMapper; import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; @@ -30,21 +31,32 @@ public class AppCfgServiceImpl implements AppCfgService { @Resource private IConfigCenter configCenter; + @Resource + private UserMapper userMapper; + @Override public Page pageAppCfgVo(PageReq page, String app) { + + List apps = userMapper.listApp(); + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); List cfgVos = new ArrayList<>(); for (KeyValue kv : keyValues) { String v = kv.getValue().toStringUtf8(); String key = kv.getKey().toStringUtf8(); - if(StringUtil.isEmpty(v)){ - configCenter.put(key, JSON.toJSONString(new AppCfgVo(key))); + String k = key.replace(ConfigConstant.appCfgPath,""); + + if(!apps.contains(k)){ + configCenter.delete(key); continue; + }else if(StringUtil.isEmpty(v)){ + v = JSON.toJSONString(new AppCfgVo(key)); + configCenter.put(key, v); } + AppCfgVo vo = JSON.parseObject(v, AppCfgVo.class); vo.setVersion(kv.getModRevision()); - String k = key.replace(ConfigConstant.appCfgPath,""); if(StringUtils.isEmpty(app)){ cfgVos.add(vo); }else{ diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java index 26f38ec..fee33fc 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.dashboard.biz.service.impl; +import cn.hutool.core.date.SystemClock; import cn.hutool.core.lang.UUID; import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageHelper; @@ -122,9 +123,13 @@ public class RuleServiceImpl implements RuleService { @Override public int save(Rules rules) { String app = rules.getApp(); - String from = ""; + + KeyValue kv = configCenter.getKv(ConfigConstant.rulePath + app); + String from = kv.getValue().toStringUtf8(); String to = JSON.toJSONString(rules); configCenter.put(ConfigConstant.rulePath + app, rules.getRules()); + + logMapper.insertSelective(new ChangeLog(app, 1, from, to,rules.getUpdateUser(), app, SystemClock.nowDate())); return 1; } -- Gitee From a62f755f4dba61a5704d8474b0d843e1dd5672e2 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Thu, 29 Oct 2020 15:42:45 +0800 Subject: [PATCH 38/59] =?UTF-8?q?app=E9=85=8D=E7=BD=AE=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/service/impl/AppCfgServiceImpl.java | 28 ++++++++++------ .../biz/service/impl/RuleServiceImpl.java | 33 +++++++++---------- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index ebbc80a..aeaa7c0 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -43,33 +43,41 @@ public class AppCfgServiceImpl implements AppCfgService { List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); List cfgVos = new ArrayList<>(); for (KeyValue kv : keyValues) { - String v = kv.getValue().toStringUtf8(); String key = kv.getKey().toStringUtf8(); - String k = key.replace(ConfigConstant.appCfgPath,""); - - if(!apps.contains(k)){ + String k = key.replace(ConfigConstant.appCfgPath, ""); + //如果某个配置所属的app已经被删了，则删除该app的配置 + if (!apps.contains(k)) { configCenter.delete(key); continue; - }else if(StringUtil.isEmpty(v)){ + } + + //取到配置信息 + String v = kv.getValue().toStringUtf8(); + //如果为空，则赋值初始化 + if (StringUtil.isEmpty(v)) { v = JSON.toJSONString(new AppCfgVo(key)); configCenter.put(key, v); } AppCfgVo vo = JSON.parseObject(v, AppCfgVo.class); vo.setVersion(kv.getModRevision()); - if(StringUtils.isEmpty(app)){ + //如果是管理员，则添加所有 + if (StringUtils.isEmpty(app)) { cfgVos.add(vo); - }else{ - if(k.equals(app)){ cfgVos.add(vo); } + } else { + //添加自己的app信息 + if (k.equals(app)) { + cfgVos.add(vo); + } } } - return PageUtil.pagination(cfgVos,page.getPageSize(),page.getPageNum()-1); + return PageUtil.pagination(cfgVos, page.getPageSize(), page.getPageNum() - 1); } @Override public AppCfgVo selectAppCfgVo(String app) { KeyValue kv = configCenter.getKv(ConfigConstant.appCfgPath + app); - if(kv == null || kv.getValue() == null){ + if (kv == null || kv.getValue() == null) { AppCfgVo ap = new AppCfgVo(app); configCenter.put(ConfigConstant.appCfgPath + app, JSON.toJSONString(ap)); return ap; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java index fee33fc..34732a3 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java @@ -53,13 +53,13 @@ public class RuleServiceImpl implements RuleService { @Override public Rules selectRules(String app) { KeyValue kv = configCenter.getKv(ConfigConstant.rulePath + app); - if(kv == null || kv.getValue() == null){ + if (kv == null || kv.getValue() == null) { return new Rules(); } String k = kv.getKey().toStringUtf8(); String v = kv.getValue().toStringUtf8(); - List rule = JSON.parseArray(v,Rule.class); - return new Rules(app,v); + List rule = JSON.parseArray(v, Rule.class); + return new Rules(app, v); } @Transactional(rollbackFor = Exception.class) @@ -68,7 +68,7 @@ public class RuleServiceImpl implements RuleService { String app = rules.getApp(); Rules oldRules = rulesMapper.select(app); String from = JSON.toJSONString(oldRules); - configCenter.put(ConfigConstant.rulePath+app, rules.getRules()); + configCenter.put(ConfigConstant.rulePath + app, rules.getRules()); String to = JSON.toJSONString(rules); logMapper.insertSelective(new ChangeLog(app, 1, from, to, rules.getUpdateUser(), app, UUID.fastUUID().toString(true))); @@ -79,7 +79,7 @@ public class RuleServiceImpl implements RuleService { @Override public Integer add(Rules rules) { String app = rules.getApp(); - configCenter.put(ConfigConstant.rulePath+app,rules.getRules()); + configCenter.put(ConfigConstant.rulePath + app, rules.getRules()); String to = JSON.toJSONString(rules); logMapper.insertSelective(new ChangeLog(app, 1, "", to, rules.getUpdateUser(), app, UUID.fastUUID().toString(true))); @@ -104,15 +104,15 @@ public class RuleServiceImpl implements RuleService { List rules = new ArrayList<>(); for (KeyValue kv : keyValues) { String v = kv.getValue().toStringUtf8(); - if(StringUtil.isEmpty(v)){ + if (StringUtil.isEmpty(v)) { continue; } String key = kv.getKey().toStringUtf8(); - String k = key.replace(ConfigConstant.rulePath,""); - if(StringUtils.isEmpty(appName)){ + String k = key.replace(ConfigConstant.rulePath, ""); + if (StringUtils.isEmpty(appName)) { rules.add(new Rules(k, v)); - }else{ - if(k.equals(appName)){ + } else { + if (k.equals(appName)) { rules.add(new Rules(k, v)); } } @@ -129,7 +129,7 @@ public class RuleServiceImpl implements RuleService { String to = JSON.toJSONString(rules); configCenter.put(ConfigConstant.rulePath + app, rules.getRules()); - logMapper.insertSelective(new ChangeLog(app, 1, from, to,rules.getUpdateUser(), app, SystemClock.nowDate())); + logMapper.insertSelective(new ChangeLog(app, 1, from, to, rules.getUpdateUser(), app, SystemClock.nowDate())); return 1; } @@ -139,14 +139,14 @@ public class RuleServiceImpl implements RuleService { List rules = new ArrayList<>(); for (KeyValue kv : keyValues) { String v = kv.getValue().toStringUtf8(); - if(StringUtil.isEmpty(v)){ + if (StringUtil.isEmpty(v)) { continue; } String key = kv.getKey().toStringUtf8(); - String appKey = key.replace(ConfigConstant.rulePath,""); + String appKey = key.replace(ConfigConstant.rulePath, ""); List rs = JSON.parseArray(v, Rule.class); for (Rule r : rs) { - rules.add(appKey+"-"+r.getKey()); + rules.add(appKey + "-" + r.getKey()); } } return rules; @@ -154,11 +154,10 @@ public class RuleServiceImpl implements RuleService { @Override public PageInfo pageRuleHitCount(PageReq pageReq, SearchReq req, String ownApp) { - PageHelper.startPage(pageReq.getPageNum(),pageReq.getPageSize()); + PageHelper.startPage(pageReq.getPageNum(), pageReq.getPageSize()); List hitCountVos = summaryMapper.listRuleHitCount(req); - return new PageInfo<>(hitCountVos); + return new PageInfo<>(hitCountVos); } - } -- Gitee From ae4834b84462b3253c3e30618502a003ac43ab56 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 29 Oct 2020 17:32:02 +0800 Subject: [PATCH 39/59] =?UTF-8?q?=E5=BA=94=E7=94=A8=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E7=9A=84=E6=97=B6=E5=80=99=20=E9=85=8D=E7=BD=AE=E8=81=94?= =?UTF-8?q?=E5=8A=A8=E4=BF=AE=E6=94=B9fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/service/impl/AppCfgServiceImpl.java | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index aeaa7c0..d9b5be3 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -2,6 +2,7 @@ package com.jd.platform.hotkey.dashboard.biz.service.impl; import com.alibaba.fastjson.JSON; import com.github.pagehelper.util.StringUtil; +import com.google.protobuf.ByteString; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; @@ -17,6 +18,7 @@ import org.springframework.util.StringUtils; import javax.annotation.Resource; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; /** * @ProjectName: hotkey @@ -40,7 +42,8 @@ public class AppCfgServiceImpl implements AppCfgService { List apps = userMapper.listApp(); - List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + List keyValues = getKVList(apps); + List cfgVos = new ArrayList<>(); for (KeyValue kv : keyValues) { String key = kv.getKey().toStringUtf8(); @@ -50,7 +53,6 @@ public class AppCfgServiceImpl implements AppCfgService { configCenter.delete(key); continue; } - //取到配置信息 String v = kv.getValue().toStringUtf8(); //如果为空，则赋值初始化 @@ -95,4 +97,25 @@ public class AppCfgServiceImpl implements AppCfgService { public void saveAppCfgVo(AppCfgVo cfg) { configCenter.put(ConfigConstant.appCfgPath + cfg.getApp(), JSON.toJSONString(cfg)); } + + + /** + * 比较和补充 + * @param apps all db apps + */ + private List getKVList(List apps){ + + List keyValues = new ArrayList<>(configCenter.getPrefix(ConfigConstant.appCfgPath)); + List etcdApps = keyValues.stream() + .map(x -> x.getKey().toStringUtf8() + .replace(ConfigConstant.appCfgPath, "")) + .collect(Collectors.toList()); + List tempList = new ArrayList<>(apps); + tempList.removeAll(etcdApps); + for (String diffApp : tempList) { + KeyValue kv = KeyValue.newBuilder().setKey(ByteString.copyFromUtf8(ConfigConstant.appCfgPath + diffApp)).build(); + keyValues.add(kv); + } + return keyValues; + } } -- Gitee From a8d9831bd1b50edc87548b00fc31d2b30697b1b1 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 29 Oct 2020 17:56:03 +0800 Subject: [PATCH 40/59] fix --- .../hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java index d9b5be3..8f59b91 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/AppCfgServiceImpl.java @@ -57,7 +57,7 @@ public class AppCfgServiceImpl implements AppCfgService { String v = kv.getValue().toStringUtf8(); //如果为空，则赋值初始化 if (StringUtil.isEmpty(v)) { - v = JSON.toJSONString(new AppCfgVo(key)); + v = JSON.toJSONString(new AppCfgVo(k)); configCenter.put(key, v); } -- Gitee From 9b9030907c95ab87c118792e1a7386a296776df3 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Fri, 30 Oct 2020 12:04:32 +0800 Subject: [PATCH 41/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=8A=A5=E8=AD=A6?= =?UTF-8?q?=E7=9A=84=E5=9B=BE=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/common/monitor/DataHandler.java | 3 ++- .../com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 3f7c617..d2dc76a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -326,7 +326,8 @@ public class DataHandler { if (type > 0) { String str = type == 1 ? "高于最大" : "低于最小"; int threshold = type == 1 ? cfg.getWarnMax() : cfg.getWarnMin(); - String content = String.format("应用：%s热点记录在%d秒内累计: %d, %s阈值: %d, 请注意", cfg.getApp(), cfg.getWarnPeriod(), count, str, threshold); + String time = LocalDateTime.now().toString().replace("T", " "); + String content = String.format("【警报】 应用：【%s】 热点记录在%d秒内累计: %d, %s阈值: %d \n【时间】：%s", cfg.getApp(), cfg.getWarnPeriod(), count, str, threshold,time); pushHandler.pushMsg(cfg.getApp(), date, content); } else { log.error(" app config error out of rang: {}", val); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java index 364fe3a..f5f650c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java @@ -99,7 +99,7 @@ public class DongDongUtil { JSONObject extend = new JSONObject(); extend.put("url", "http://hotkey.jd.com"); - extend.put("pic", "https://img14.360buyimg.com/imagetools/jfs/t1/144156/7/8194/7263/5f5b34baEdfd90a49/a15144ab447bfbf2.png"); + /* extend.put("pic", "https://img14.360buyimg.com/imagetools/jfs/t1/144156/7/8194/7263/5f5b34baEdfd90a49/a15144ab447bfbf2.png");*/ json.put("extend", extend); params.add(new BasicNameValuePair("jsonMsg",JSON.toJSONString(json))); -- Gitee From 7ce9a610772ba70bd5cf766cf916c4749b6399ff Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Mon, 2 Nov 2020 11:36:52 +0800 Subject: [PATCH 42/59] =?UTF-8?q?=E5=A2=9E=E5=8A=A0log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/common/monitor/DataHandler.java | 2 +- .../jd/platform/hotkey/dashboard/warn/AccessTokenTask.java | 1 + .../platform/hotkey/dashboard/warn/DongDongApiManager.java | 5 ++++- .../com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java | 4 ++-- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index d2dc76a..53c4427 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -330,7 +330,7 @@ public class DataHandler { String content = String.format("【警报】 应用：【%s】 热点记录在%d秒内累计: %d, %s阈值: %d \n【时间】：%s", cfg.getApp(), cfg.getWarnPeriod(), count, str, threshold,time); pushHandler.pushMsg(cfg.getApp(), date, content); } else { - log.error(" app config error out of rang: {}", val); + log.info("app：{} count:{}, int normal rang: {}",cfg.getApp(), count, val); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/AccessTokenTask.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/AccessTokenTask.java index 5f75344..75d0f73 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/AccessTokenTask.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/AccessTokenTask.java @@ -23,6 +23,7 @@ public class AccessTokenTask { @Scheduled(cron = "0 0 0/1 ? * *") public void updateAccessToken(){ try { + log.info("=== time to refreshAccessSignature ==="); apiManager.refreshAccessSignature(); }catch (Exception e){ log.error("refreshAccessSignature error:",e); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java index 25d2dd2..65c2f08 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.dashboard.warn; +import com.alibaba.fastjson.JSON; import com.github.rholder.retry.*; import com.jd.platform.hotkey.dashboard.biz.service.IBizAccessTokenService; import com.jd.platform.hotkey.dashboard.model.BizAccessToken; @@ -39,13 +40,15 @@ public class DongDongApiManager { try { String accessToken = dongUtil.grant(); List tokens = accessTokenService.selectBizAccessTokenList(new BizAccessToken()); + log.info("TASK grant accessToken:{}, 数据库查询tokenList:{}",accessToken, JSON.toJSONString(tokens)); if(tokens.size()>0){ BizAccessToken token = tokens.get(0); token.setToken(accessToken); token.setUpdatedBy("system"); token.setUpdatedTime(new Date()); log.info("updateBizAccessToken={}", accessToken); - accessTokenService.updateBizAccessToken(token); + int row = accessTokenService.updateBizAccessToken(token); + log.info("TASK refresh accessToken 影响行数：{}, 修改对象：{}",row, JSON.toJSONString(token)); }else{ BizAccessToken token=new BizAccessToken(); token.setToken(accessToken); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java index f5f650c..e3f635e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java @@ -57,7 +57,7 @@ public class DongDongUtil { CloseableHttpResponse response = httpclient.execute(httpPost); String res = EntityUtils.toString(response.getEntity()); - + log.info("TASK grantAccessSign param:{} result:{}",JSON.toJSONString(params),res); //返回结果对象AccessSignatureResult AccessSignatureResult result = JSON.parseObject(res, AccessSignatureResult.class); if (result.getCode() == 230031) { @@ -110,7 +110,7 @@ public class DongDongUtil { httpPost.setEntity(new UrlEncodedFormEntity(params,"UTF-8")); CloseableHttpResponse response = httpclient.execute(httpPost); String res = EntityUtils.toString(response.getEntity()); - log.info("发送报警信息：{}", res); + log.info("发送报警信息参数：{} 结果：{}", JSON.toJSONString(params), res); //返回结果对象AccessSignatureResult MessagePushResult result = JSON.parseObject(res, MessagePushResult.class); return result.getCode() == 230070; -- Gitee From 3b80df07fe00739187d38b2178439ab382c8042b Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Mon, 2 Nov 2020 14:01:54 +0800 Subject: [PATCH 43/59] =?UTF-8?q?=E6=8A=A5=E8=AD=A6=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/controller/KeyController.java | 6 +- .../biz/mapper/BizAccessTokenMapper.java | 62 ---------- .../biz/service/IBizAccessTokenService.java | 55 --------- .../impl/BizAccessTokenServiceImpl.java | 80 ------------ .../biz/service/impl/KeyServiceImpl.java | 22 ++-- .../dashboard/common/domain/req/ChartReq.java | 6 +- .../common/domain/req/SearchReq.java | 4 +- .../dashboard/common/monitor/DataHandler.java | 50 ++++---- .../dashboard/model/BizAccessToken.java | 116 ------------------ .../hotkey/dashboard/util/CommonUtil.java | 17 ++- .../util/{DateUtil.java => DateUtils.java} | 2 +- .../hotkey/dashboard/util/TwoTuple.java | 58 --------- .../dashboard/warn/AccessTokenTask.java | 31 ----- .../dashboard/warn/DongDongApiManager.java | 76 +++--------- .../hotkey/dashboard/warn/DongDongToken.java | 10 ++ .../biz/mapper/BizAccessTokenMapper.xml | 73 ----------- .../dashboard/biz/mapper/KeyRecordMapper.xml | 1 + .../dashboard/biz/mapper/SummaryMapper.xml | 1 + .../worker/netty/filter/HotKeyFilter.java | 5 +- 19 files changed, 89 insertions(+), 586 deletions(-) delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/BizAccessTokenMapper.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/IBizAccessTokenService.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/BizAccessTokenServiceImpl.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/BizAccessToken.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/{DateUtil.java => DateUtils.java} (99%) delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/TwoTuple.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/AccessTokenTask.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongToken.java delete mode 100644 dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/BizAccessTokenMapper.xml diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java index 281c26a..2c0b36f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java @@ -15,7 +15,7 @@ import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.biz.service.KeyService; -import com.jd.platform.hotkey.dashboard.util.DateUtil; +import com.jd.platform.hotkey.dashboard.util.DateUtils; import com.jd.platform.hotkey.dashboard.util.ExcelUtil; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; @@ -143,10 +143,10 @@ public class KeyController extends BaseController { public void export(HttpServletResponse resp,String startTime,String endTime,String app,String key){ SearchReq req = new SearchReq(); if(StringUtil.isNotEmpty(startTime)){ - req.setStartTime(DateUtil.strToDate(startTime)); + req.setStartTime(DateUtils.strToDate(startTime)); } if(StringUtil.isNotEmpty(endTime)){ - req.setEndTime(DateUtil.strToDate(endTime)); + req.setEndTime(DateUtils.strToDate(endTime)); } req.setApp(app); req.setKey(key); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/BizAccessTokenMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/BizAccessTokenMapper.java deleted file mode 100644 index 2d4db7f..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/mapper/BizAccessTokenMapper.java +++ /dev/null @@ -1,62 +0,0 @@ -package com.jd.platform.hotkey.dashboard.biz.mapper; - -import com.jd.platform.hotkey.dashboard.model.BizAccessToken; -import org.apache.ibatis.annotations.Mapper; - -import java.util.List; - -/** - * 咚咚消息tokenMapper接口 - * - * @date 2020-05-21 - */ -@Mapper -public interface BizAccessTokenMapper { - /** - * 查询咚咚消息token - * - * @param id 咚咚消息tokenID - * @return 咚咚消息token - */ - public BizAccessToken selectBizAccessTokenById(Long id); - - /** - * 查询咚咚消息token列表 - * - * @param bizAccessToken 咚咚消息token - * @return 咚咚消息token集合 - */ - public List selectBizAccessTokenList(BizAccessToken bizAccessToken); - - /** - * 新增咚咚消息token - * - * @param bizAccessToken 咚咚消息token - * @return 结果 - */ - public int insertBizAccessToken(BizAccessToken bizAccessToken); - - /** - * 修改咚咚消息token - * - * @param bizAccessToken 咚咚消息token - * @return 结果 - */ - public int updateBizAccessToken(BizAccessToken bizAccessToken); - - /** - * 删除咚咚消息token - * - * @param id 咚咚消息tokenID - * @return 结果 - */ - public int deleteBizAccessTokenById(Long id); - - /** - * 批量删除咚咚消息token - * - * @param ids 需要删除的数据ID - * @return 结果 - */ - public int deleteBizAccessTokenByIds(String[] ids); -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/IBizAccessTokenService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/IBizAccessTokenService.java deleted file mode 100644 index d5c90a6..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/IBizAccessTokenService.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.jd.platform.hotkey.dashboard.biz.service; - - -import com.jd.platform.hotkey.dashboard.model.BizAccessToken; - -import java.util.List; - -/** - * 咚咚消息tokenService接口 - * - * @author ruoyi - * @date 2020-05-21 - */ -public interface IBizAccessTokenService -{ - /** - * 查询咚咚消息token - * - * @param id 咚咚消息tokenID - * @return 咚咚消息token - */ - public BizAccessToken selectBizAccessTokenById(Long id); - - /** - * 查询咚咚消息token列表 - * - * @param bizAccessToken 咚咚消息token - * @return 咚咚消息token集合 - */ - public List selectBizAccessTokenList(BizAccessToken bizAccessToken); - - /** - * 新增咚咚消息token - * - * @param bizAccessToken 咚咚消息token - * @return 结果 - */ - public int insertBizAccessToken(BizAccessToken bizAccessToken); - - /** - * 修改咚咚消息token - * - * @param bizAccessToken 咚咚消息token - * @return 结果 - */ - public int updateBizAccessToken(BizAccessToken bizAccessToken); - - /** - * 删除咚咚消息token信息 - * - * @param id 咚咚消息tokenID - * @return 结果 - */ - public int deleteBizAccessTokenById(Long id); -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/BizAccessTokenServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/BizAccessTokenServiceImpl.java deleted file mode 100644 index 2cfd003..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/BizAccessTokenServiceImpl.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.jd.platform.hotkey.dashboard.biz.service.impl; - -import com.jd.platform.hotkey.dashboard.biz.mapper.BizAccessTokenMapper; -import com.jd.platform.hotkey.dashboard.biz.service.IBizAccessTokenService; -import com.jd.platform.hotkey.dashboard.model.BizAccessToken; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import javax.annotation.Resource; -import java.util.List; - -/** - * 咚咚消息tokenService业务层处理 - * - * @author ruoyi - * @date 2020-05-21 - */ -@Service -public class BizAccessTokenServiceImpl implements IBizAccessTokenService { - - - @Resource - private BizAccessTokenMapper bizAccessTokenMapper; - - /** - * 查询咚咚消息token - * - * @param id 咚咚消息tokenID - * @return 咚咚消息token - */ - @Override - public BizAccessToken selectBizAccessTokenById(Long id){ - return bizAccessTokenMapper.selectBizAccessTokenById(id); - } - - /** - * 查询咚咚消息token列表 - * - * @param bizAccessToken 咚咚消息token - * @return 咚咚消息token - */ - @Override - public List selectBizAccessTokenList(BizAccessToken bizAccessToken){ - return bizAccessTokenMapper.selectBizAccessTokenList(bizAccessToken); - } - - /** - * 新增咚咚消息token - * - * @param bizAccessToken 咚咚消息token - * @return 结果 - */ - @Override - public int insertBizAccessToken(BizAccessToken bizAccessToken){ - return bizAccessTokenMapper.insertBizAccessToken(bizAccessToken); - } - - /** - * 修改咚咚消息token - * - * @param bizAccessToken 咚咚消息token - * @return 结果 - */ - @Override - public int updateBizAccessToken(BizAccessToken bizAccessToken){ - return bizAccessTokenMapper.updateBizAccessToken(bizAccessToken); - } - - /** - * 删除咚咚消息token信息 - * - * @param id 咚咚消息tokenID - * @return 结果 - */ - @Override - public int deleteBizAccessTokenById(Long id) - { - return bizAccessTokenMapper.deleteBizAccessTokenById(id); - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java index 42ebd19..c7a5f0a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java @@ -26,7 +26,7 @@ import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.biz.service.KeyService; import com.jd.platform.hotkey.dashboard.biz.service.RuleService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; -import com.jd.platform.hotkey.dashboard.util.DateUtil; +import com.jd.platform.hotkey.dashboard.util.DateUtils; import com.jd.platform.hotkey.dashboard.util.PageUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import org.slf4j.Logger; @@ -79,12 +79,12 @@ public class KeyServiceImpl implements KeyService { } req.setApp(null); LocalDateTime now = LocalDateTime.now(); - req.setEndTime(req.getEndTime() == null ? DateUtil.ldtToDate(now) : req.getEndTime()); + req.setEndTime(req.getEndTime() == null ? DateUtils.ldtToDate(now) : req.getEndTime()); List rules = ruleService.listRules(null); if (type == 4) { - LocalDateTime st = req.getStartTime() == null ? now.minusMinutes(31) : DateUtil.dateToLdt(req.getStartTime()); - req.setStartTime(DateUtil.ldtToDate(st)); - LocalDateTime et = DateUtil.dateToLdt(req.getEndTime()); + LocalDateTime st = req.getStartTime() == null ? now.minusMinutes(31) : DateUtils.dateToLdt(req.getStartTime()); + req.setStartTime(DateUtils.ldtToDate(st)); + LocalDateTime et = DateUtils.dateToLdt(req.getEndTime()); boolean longTime = Duration.between(st, et).toHours() > 2; req.setType(longTime ? 6 : 5); List list = statisticsMapper.listOrderByTime(req); @@ -93,17 +93,17 @@ public class KeyServiceImpl implements KeyService { if (type == 5) { LocalDateTime startTime = now.minusMinutes(31); - req.setStartTime(DateUtil.ldtToDate(startTime)); + req.setStartTime(DateUtils.ldtToDate(startTime)); List list = statisticsMapper.listOrderByTime(req); return CommonUtil.processData(startTime, now, list, true, rules, app); } else if (type == 6) { LocalDateTime startTime2 = now.minusHours(25); - req.setStartTime(DateUtil.ldtToDate(startTime2)); + req.setStartTime(DateUtils.ldtToDate(startTime2)); List list2 = statisticsMapper.listOrderByTime(req); return CommonUtil.processData(startTime2, now, list2, false, rules, app); } else { LocalDateTime startTime3 = now.minusDays(7).minusHours(1); - req.setStartTime(DateUtil.ldtToDate(startTime3)); + req.setStartTime(DateUtils.ldtToDate(startTime3)); req.setType(6); List list3 = statisticsMapper.listOrderByTime(req); return CommonUtil.processData(startTime3, now, list3, false, rules, app); @@ -138,7 +138,7 @@ public class KeyServiceImpl implements KeyService { int hours = 6; // 默认查询6小时内的数据 SearchReq req = new SearchReq(); - req.setStartTime(DateUtil.preTime(hours)); + req.setStartTime(DateUtils.preTime(hours)); req.setEndTime(new Date()); List statistics = statisticsMapper.listStatistics(req); // 获取data Y轴 @@ -215,7 +215,7 @@ public class KeyServiceImpl implements KeyService { Map map = new HashMap<>(10); Map> listMap = statistics.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); for (Map.Entry> m : listMap.entrySet()) { - int start = DateUtil.preHoursInt(5); + int start = DateUtils.preHoursInt(5); map.put(m.getKey(), new int[hours]); int[] data = map.get(m.getKey()); int tmp = 0; @@ -244,7 +244,7 @@ public class KeyServiceImpl implements KeyService { private void checkParam(SearchReq req) { if (req.getStartTime() == null || req.getEndTime() == null) { - req.setStartTime(DateUtil.preTime(5)); + req.setStartTime(DateUtils.preTime(5)); req.setEndTime(new Date()); } /* long day = (req.getEndTime().getTime() - req.getStartTime().getTime()) / 86400000; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java index 5154e43..1df046b 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java @@ -1,6 +1,6 @@ package com.jd.platform.hotkey.dashboard.common.domain.req; -import com.jd.platform.hotkey.dashboard.util.DateUtil; +import com.jd.platform.hotkey.dashboard.util.DateUtils; import java.io.Serializable; import java.time.LocalDateTime; @@ -28,8 +28,8 @@ public class ChartReq implements Serializable { } public ChartReq(LocalDateTime st, LocalDateTime et, Integer limit) { - this.startTime = DateUtil.ldtToDate(st); - this.endTime = DateUtil.ldtToDate(et); + this.startTime = DateUtils.ldtToDate(st); + this.endTime = DateUtils.ldtToDate(et); this.limit = limit; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java index a63d987..8ee7f49 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java @@ -1,7 +1,7 @@ package com.jd.platform.hotkey.dashboard.common.domain.req; import com.fasterxml.jackson.annotation.JsonFormat; -import com.jd.platform.hotkey.dashboard.util.DateUtil; +import com.jd.platform.hotkey.dashboard.util.DateUtils; import java.io.Serializable; import java.time.LocalDateTime; @@ -82,7 +82,7 @@ public class SearchReq implements Serializable { } public SearchReq(LocalDateTime st) { - this.startTime = DateUtil.ldtToDate(st); + this.startTime = DateUtils.ldtToDate(st); this.endTime = new Date(); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index d2dc76a..33f40d1 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -2,6 +2,8 @@ package com.jd.platform.hotkey.dashboard.common.monitor; import cn.hutool.core.collection.CollectionUtil; +import cn.hutool.core.date.DateTime; +import cn.hutool.core.date.DateUtil; import com.alibaba.fastjson.JSON; import com.google.common.collect.Queues; import com.ibm.etcd.api.KeyValue; @@ -18,7 +20,7 @@ import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; -import com.jd.platform.hotkey.dashboard.util.DateUtil; +import com.jd.platform.hotkey.dashboard.util.DateUtils; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -143,9 +145,9 @@ public class DataHandler { public void offlineStatistics() { try { LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); + Date nowTime = DateUtils.ldtToDate(now); + int day = DateUtils.nowDay(now); + int hour = DateUtils.nowHour(now); SearchReq preHour = new SearchReq(now.minusHours(1)); List records = keyRecordMapper.maxHotKey(preHour); if (records.size() != 0) { @@ -188,10 +190,10 @@ public class DataHandler { public void offlineStatisticsRule() { try { LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtil.ldtToDate(now); - int day = DateUtil.nowDay(now); - int hour = DateUtil.nowHour(now); - int minus = DateUtil.nowMinus(now); + Date nowTime = DateUtils.ldtToDate(now); + int day = DateUtils.nowDay(now); + int hour = DateUtils.nowHour(now); + int minus = DateUtils.nowMinus(now); List records = keyRecordMapper.statisticsByRule(new SearchReq(now.minusMinutes(1))); if (records.size() == 0) { @@ -219,26 +221,30 @@ public class DataHandler { /** * 每天根据app的配置清理过期数据 */ - @Scheduled(cron = "0 0 1 * * ?") +// @Scheduled(cron = "0 0 3 * * ?") + @Scheduled(fixedRate = 5000) //每隔5秒删1万 public void clearExpireData() { - try { - LocalDateTime now = LocalDateTime.now(); - List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); - for (KeyValue kv : keyValues) { + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + for (KeyValue kv : keyValues) { + try { String val = kv.getValue().toStringUtf8(); AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); String app = cfg.getApp(); - Date expireDate = DateUtil.ldtToDate(now.minusDays(cfg.getDataTtl())); - summaryMapper.clearExpireData(app, expireDate); - keyRecordMapper.clearExpireData(app, expireDate); - statisticsMapper.clearExpireData(app, expireDate); + //保存几天 + Integer days = cfg.getDataTtl(); + DateTime dateTime = DateUtil.offsetDay(new Date(), -days); + + + summaryMapper.clearExpireData(app, dateTime); + keyRecordMapper.clearExpireData(app, dateTime); + statisticsMapper.clearExpireData(app, dateTime); + } catch (Exception e) { + e.printStackTrace(); } - } catch (Exception e) { - e.printStackTrace(); } - } + } /** * 每10秒检测一次热点记录 用于监控报警 @@ -327,10 +333,8 @@ public class DataHandler { String str = type == 1 ? "高于最大" : "低于最小"; int threshold = type == 1 ? cfg.getWarnMax() : cfg.getWarnMin(); String time = LocalDateTime.now().toString().replace("T", " "); - String content = String.format("【警报】 应用：【%s】 热点记录在%d秒内累计: %d, %s阈值: %d \n【时间】：%s", cfg.getApp(), cfg.getWarnPeriod(), count, str, threshold,time); + String content = String.format("【警报】 应用：【%s】 热点记录在%d秒内累计: %d, %s阈值: %d \n【时间】：%s", cfg.getApp(), cfg.getWarnPeriod(), count, str, threshold, time); pushHandler.pushMsg(cfg.getApp(), date, content); - } else { - log.error(" app config error out of rang: {}", val); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/BizAccessToken.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/BizAccessToken.java deleted file mode 100644 index 98c0c4d..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/BizAccessToken.java +++ /dev/null @@ -1,116 +0,0 @@ -package com.jd.platform.hotkey.dashboard.model; - -import org.apache.commons.lang3.builder.ToStringBuilder; -import org.apache.commons.lang3.builder.ToStringStyle; - -import java.io.Serializable; -import java.util.Date; - -/** - * 咚咚消息token对象 biz_access_token - * @author ruoyi - * @date 2020-05-21 - */ -public class BizAccessToken implements Serializable { - private static final long serialVersionUID = 1L; - - public String APP = "2"; - - /** ID */ - private Long id; - - /** Token */ - private String token; - - /** flag */ - private String flag; - - /** 创建人 */ - private String createdBy; - - /** 创建时间 */ - private Date createdTime; - - /** 更新人 */ - private String updatedBy; - - /** 更新时间 */ - private Date updatedTime; - - public void setId(Long id) - { - this.id = id; - } - - public Long getId() - { - return id; - } - public void setToken(String token) - { - this.token = token; - } - - public String getToken() - { - return token; - } - public void setFlag(String flag) - { - this.flag = flag; - } - - public String getFlag() - { - return flag; - } - public void setCreatedBy(String createdBy) - { - this.createdBy = createdBy; - } - - public String getCreatedBy() - { - return createdBy; - } - public void setCreatedTime(Date createdTime) - { - this.createdTime = createdTime; - } - - public Date getCreatedTime() - { - return createdTime; - } - public void setUpdatedBy(String updatedBy) - { - this.updatedBy = updatedBy; - } - - public String getUpdatedBy() - { - return updatedBy; - } - public void setUpdatedTime(Date updatedTime) - { - this.updatedTime = updatedTime; - } - - public Date getUpdatedTime() - { - return updatedTime; - } - - @Override - public String toString() { - return new ToStringBuilder(this, ToStringStyle.MULTI_LINE_STYLE) - .append("id", getId()) - .append("token", getToken()) - .append("flag", getFlag()) - .append("createdBy", getCreatedBy()) - .append("createdTime", getCreatedTime()) - .append("updatedBy", getUpdatedBy()) - .append("updatedTime", getUpdatedTime()) - .toString(); - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 4ec40fc..deda3f8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.dashboard.util; -import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.model.Summary; @@ -118,10 +117,10 @@ public class CommonUtil { Duration duration = Duration.between(st, et); long passTime = isMinute ? duration.toMinutes() : duration.toHours(); Map timeCountMap = new TreeMap<>(); - String pattern = isMinute ? DateUtil.PATTERN_MINUS : DateUtil.PATTERN_HOUR; + String pattern = isMinute ? DateUtils.PATTERN_MINUS : DateUtils.PATTERN_HOUR; for (int i = 1; i < passTime; i++) { - int time = DateUtil.reviseTime(st, i, isMinute ? 1 : 2); - xAxisSet.add(DateUtil.formatTime(time, pattern)); + int time = DateUtils.reviseTime(st, i, isMinute ? 1 : 2); + xAxisSet.add(DateUtils.formatTime(time, pattern)); timeCountMap.put(time, null); } Map> ruleStatsMap = listGroup(list); @@ -175,12 +174,12 @@ public class CommonUtil { String app = args[0]; String rule = args[1]; String hitTime = args[2]; - Date time = DateUtil.strToDate(hitTime); + Date time = DateUtils.strToDate(hitTime); assert time != null; - LocalDateTime ldt = DateUtil.dateToLdt(time); - int day = DateUtil.nowDay(ldt); - int hour = DateUtil.nowHour(ldt); - int minus = DateUtil.nowMinus(ldt); + LocalDateTime ldt = DateUtils.dateToLdt(time); + int day = DateUtils.nowDay(ldt); + int hour = DateUtils.nowHour(ldt); + int minus = DateUtils.nowMinus(ldt); long seconds = time.getTime() / 1000; String[] counts = map.get(key).split("-"); int hitCount = Integer.parseInt(counts[0]); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java similarity index 99% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java index 8232567..a854aba 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java @@ -10,7 +10,7 @@ import java.util.Date; /** * @author liyunfeng31 */ -public class DateUtil { +public class DateUtils { public static final String PATTERN_SECONDS="yyMMddHHmmss"; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/TwoTuple.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/TwoTuple.java deleted file mode 100644 index 0fdd807..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/TwoTuple.java +++ /dev/null @@ -1,58 +0,0 @@ -package com.jd.platform.hotkey.dashboard.util; - -import java.io.Serializable; - -/** - * TwoTupleTwoTupleTwoTuple - * @date 2020-02-24 - * @author wuweifeng wrote on 2020-02-24 - * @version 1.0 - */ -public class TwoTuple implements Serializable { - private static final long serialVersionUID = 1L; - /** - * first - */ - private K first; - /** - * second - */ - private V second; - - /** - * TwoTuple - */ - public TwoTuple() { - } - - /** - * getFirst - * @return first - */ - public K getFirst() { - return this.first; - } - - public void setFirst(K first) { - this.first = first; - } - - /** - * second - * @return second - */ - public V getSecond() { - return this.second; - } - - public void setSecond(V second) { - this.second = second; - } - - @Override - public String toString() { - StringBuilder strBuilder = new StringBuilder(); - strBuilder.append("[first = ").append(this.first).append(",").append("second = ").append("]"); - return strBuilder.toString(); - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/AccessTokenTask.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/AccessTokenTask.java deleted file mode 100644 index 5f75344..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/AccessTokenTask.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; - -/** - * @author liyunfeng31 - */ -@Component("accessTokenTask") -public class AccessTokenTask { - - @Resource - private DongDongApiManager apiManager; - - - private Logger log = LoggerFactory.getLogger(getClass()); - - - @Scheduled(cron = "0 0 0/1 ? * *") - public void updateAccessToken(){ - try { - apiManager.refreshAccessSignature(); - }catch (Exception e){ - log.error("refreshAccessSignature error:",e); - } - } -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java index 25d2dd2..ce3b04f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java @@ -1,21 +1,14 @@ package com.jd.platform.hotkey.dashboard.warn; -import com.github.rholder.retry.*; -import com.jd.platform.hotkey.dashboard.biz.service.IBizAccessTokenService; -import com.jd.platform.hotkey.dashboard.model.BizAccessToken; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thymeleaf.util.StringUtils; +import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.util.Date; import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.TimeUnit; /** * @author lihongliang32 @@ -28,65 +21,32 @@ public class DongDongApiManager { private DongDongUtil dongUtil; - private Logger log = LoggerFactory.getLogger(getClass()); + private Logger log = LoggerFactory.getLogger(getClass()); - - @Autowired - private IBizAccessTokenService accessTokenService; - - public boolean refreshAccessSignature() { - Callable callable = () -> { - try { - String accessToken = dongUtil.grant(); - List tokens = accessTokenService.selectBizAccessTokenList(new BizAccessToken()); - if(tokens.size()>0){ - BizAccessToken token = tokens.get(0); - token.setToken(accessToken); - token.setUpdatedBy("system"); - token.setUpdatedTime(new Date()); - log.info("updateBizAccessToken={}", accessToken); - accessTokenService.updateBizAccessToken(token); - }else{ - BizAccessToken token=new BizAccessToken(); - token.setToken(accessToken); - token.setCreatedBy("system"); - token.setCreatedTime(new Date()); - log.info("insertBizAccessToken={}", accessToken); - accessTokenService.insertBizAccessToken(token); - } - return true; - }catch (Exception e){ - return false; - } - }; - boolean result = false; - Retryer retryer = RetryerBuilder.newBuilder() - .retryIfRuntimeException() - .withWaitStrategy(WaitStrategies.fibonacciWait(100, 2, TimeUnit.MINUTES)) - .withStopStrategy(StopStrategies.stopAfterAttempt(5)) - .build(); + /** + * 启动时获取token，每隔1小时刷新一次token + */ + @Scheduled(cron = "0 0 * * * ?") + @PostConstruct + public void updateAccessToken() { try { - result = retryer.call(callable); - } catch (RetryException | ExecutionException e) { - log.error("DongdongOpenAPIManagerImpl::refreshAccessSignature Exception!", e); + String accessToken = dongUtil.grant(); + log.info("刷新token：" + accessToken); + DongDongToken.TOKEN = accessToken; + } catch (Exception e) { + log.error("refreshAccessSignature error:", e); } - return result; } public boolean push(String title, String content, List erpList) { - List tokens = accessTokenService.selectBizAccessTokenList(new BizAccessToken()); - if(tokens.size() == 0){ - log.error("没有查询到有效的token！"); - return false; - } - String accessToken = tokens.get(0).getToken(); - if(StringUtils.isEmpty(accessToken)){ + String accessToken = DongDongToken.TOKEN; + if (StringUtils.isEmpty(accessToken)) { return false; } try { - return dongUtil.push(title,content,accessToken, erpList); - }catch (Exception e){ + return dongUtil.push(title, content, accessToken, erpList); + } catch (Exception e) { return false; } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongToken.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongToken.java new file mode 100644 index 0000000..b3cd438 --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongToken.java @@ -0,0 +1,10 @@ +package com.jd.platform.hotkey.dashboard.warn; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-11-02 + */ +public class DongDongToken { + public static String TOKEN = ""; +} diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/BizAccessTokenMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/BizAccessTokenMapper.xml deleted file mode 100644 index 07bf818..0000000 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/BizAccessTokenMapper.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - - - - - - - - - - - - - - select id, token, flag, CREATED_BY, CREATED_TIME, UPDATED_BY, UPDATED_TIME from biz_access_token - - - - - - - - insert into biz_access_token - - token, - flag, - CREATED_BY, - CREATED_TIME, - UPDATED_BY, - UPDATED_TIME, - - - #{token}, - #{flag}, - #{createdBy}, - #{createdTime}, - #{updatedBy}, - #{updatedTime}, - - - - - update biz_access_token - - token = #{token}, - flag = #{flag}, - CREATED_BY = #{createdBy}, - CREATED_TIME = #{createdTime}, - UPDATED_BY = #{updatedBy}, - UPDATED_TIME = #{updatedTime}, - - where id = #{id} - - - - delete from biz_access_token where id = #{id} - - \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml index 82274d2..eef4c75 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml @@ -217,6 +217,7 @@ delete from hk_key_record where app_name = #{app} + limit 10000 \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.xml index de74e50..6cae253 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/SummaryMapper.xml @@ -55,6 +55,7 @@ delete from hk_summary where app = #{app} + limit 10000 diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java index e362c18..1109a90 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/HotKeyFilter.java @@ -8,6 +8,7 @@ import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.NettyIpUtil; import com.jd.platform.hotkey.worker.keydispatcher.KeyProducer; import com.jd.platform.hotkey.worker.netty.holder.WhiteListHolder; +import com.jd.platform.hotkey.worker.starters.EtcdStarter; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -61,7 +62,9 @@ public class HotKeyFilter implements INettyMsgFilter { } long timeOut = now - model.getCreateTime(); if (timeOut > 1000) { - logger.info("key timeout " + timeOut + ", from ip : " + NettyIpUtil.clientIp(ctx)); + if (EtcdStarter.LOGGER_ON) { + logger.info("key timeout " + timeOut + ", from ip : " + NettyIpUtil.clientIp(ctx)); + } } keyProducer.push(model, now); } -- Gitee From 92b3da23ef7aa0585b4e287a807d6ee79ffe4dbb Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Mon, 2 Nov 2020 19:01:43 +0800 Subject: [PATCH 44/59] =?UTF-8?q?=E6=8A=A5=E8=AD=A6=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/common/monitor/DataHandler.java | 42 ++++++++++--------- .../dashboard/common/monitor/PushHandler.java | 24 ++++++----- .../dashboard/warn/DongDongApiManager.java | 24 +++++++---- 3 files changed, 51 insertions(+), 39 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 33f40d1..fd27f2f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -27,15 +27,11 @@ import org.slf4j.LoggerFactory; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; +import java.util.*; +import java.util.concurrent.*; @Component public class DataHandler { @@ -249,19 +245,24 @@ public class DataHandler { /** * 每10秒检测一次热点记录 用于监控报警 */ - @Scheduled(initialDelay = 30000, fixedRate = 10000) + @PostConstruct public void scanRecordForMonitor() { - try { - SearchReq req = new SearchReq(); - Date date = new Date(); - req.setEndTime(date); - List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); - for (KeyValue kv : keyValues) { - queryRecordAndCheck(kv, req, date); + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 + scheduledExecutorService.scheduleAtFixedRate(() -> { + try { + SearchReq req = new SearchReq(); + Date date = new Date(); + req.setEndTime(date); + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + for (KeyValue kv : keyValues) { + queryRecordAndCheck(kv, req, date); + } + } catch (Exception e) { + e.printStackTrace(); } - } catch (Exception e) { - e.printStackTrace(); - } + }, 2, 10, TimeUnit.SECONDS); + } @@ -321,7 +322,10 @@ public class DataHandler { req.setStartTime(new Date(date.getTime() - cfg.getWarnPeriod() * 1000)); int count = keyRecordMapper.countKeyRecord(req); - log.info("应用app:{}, 记录count:{}, 统计时间Period:{}", cfg.getApp(), count, cfg.getWarnPeriod()); + //抽样2%打印 + if (count > 0 && Math.abs(new Random().nextInt()) % 50 == 0) { + log.info("应用app:{}, 记录count:{}, 统计时间Period:{}", cfg.getApp(), count, cfg.getWarnPeriod()); + } int type = 0; if (count >= cfg.getWarnMax()) { type = 1; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java index 1b926de..fdb022d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java @@ -7,6 +7,7 @@ import com.jd.platform.hotkey.dashboard.warn.DongDongApiManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; + import javax.annotation.Resource; import java.util.Date; import java.util.List; @@ -35,13 +36,13 @@ public class PushHandler { /** * 拦截重复报警间隔 2分钟 */ - private static final long INTERVAL = 2*60*1000L; + private static final long INTERVAL = 2 * 60 * 1000L; /** * app-time 用于app存储报警时间 做拦截 */ - private static Map appIntervalMap = new ConcurrentHashMap<>(); + private Map appIntervalMap = new ConcurrentHashMap<>(); @Resource @@ -56,28 +57,29 @@ public class PushHandler { */ public void pushMsg(String app, Date msgTime, String content) { boolean send = check(app, msgTime.getTime()); - if(send){ - logger.info("Warn PushMsg content:{}",content); + if (send) { + logger.info("Warn PushMsg content:{}", content); List erpList = userMapper.listErpByApp(app); - apiManager.push(TITLE,content,erpList); + apiManager.push(TITLE, content, erpList); } } /** * 防止重复发送警报 + * * @param warnApp app * @param msgTime time * @return result */ - private synchronized boolean check(String warnApp, Long msgTime){ + private synchronized boolean check(String warnApp, Long msgTime) { Long maxTime = appIntervalMap.get(warnApp); - if(maxTime == null){ - appIntervalMap.put(warnApp,msgTime+INTERVAL); + if (maxTime == null) { + appIntervalMap.put(warnApp, msgTime + INTERVAL); return true; - }else{ - if(msgTime > maxTime){ - appIntervalMap.put(warnApp,msgTime+INTERVAL); + } else { + if (msgTime >= maxTime) { + appIntervalMap.put(warnApp, msgTime + INTERVAL); return true; } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java index ce3b04f..36f6b7c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java @@ -2,13 +2,15 @@ package com.jd.platform.hotkey.dashboard.warn; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.thymeleaf.util.StringUtils; import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; /** * @author lihongliang32 @@ -26,16 +28,20 @@ public class DongDongApiManager { /** * 启动时获取token，每隔1小时刷新一次token */ - @Scheduled(cron = "0 0 * * * ?") @PostConstruct public void updateAccessToken() { - try { - String accessToken = dongUtil.grant(); - log.info("刷新token：" + accessToken); - DongDongToken.TOKEN = accessToken; - } catch (Exception e) { - log.error("refreshAccessSignature error:", e); - } + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 + scheduledExecutorService.scheduleAtFixedRate(() -> { + try { + String accessToken = dongUtil.grant(); + log.info("刷新token：" + accessToken); + DongDongToken.TOKEN = accessToken; + } catch (Exception e) { + log.error("refreshAccessSignature error:", e); + } + }, 0, 60, TimeUnit.MINUTES); + } -- Gitee From 5b4972c662f2fc32a30150bb069607d1bd61527b Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 3 Nov 2020 11:59:34 +0800 Subject: [PATCH 45/59] =?UTF-8?q?=E5=A4=A7=E7=9B=98=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/common/monitor/DataHandler.java | 177 +++++++++--------- 1 file changed, 92 insertions(+), 85 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index fd27f2f..2f49b82 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -24,7 +24,6 @@ import com.jd.platform.hotkey.dashboard.util.DateUtils; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -137,79 +136,85 @@ public class DataHandler { /** * 每小时 统计一次record 表 结果记录到统计表 */ - @Scheduled(cron = "0 0 * * * ?") + @PostConstruct public void offlineStatistics() { - try { - LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtils.ldtToDate(now); - int day = DateUtils.nowDay(now); - int hour = DateUtils.nowHour(now); - SearchReq preHour = new SearchReq(now.minusHours(1)); - List records = keyRecordMapper.maxHotKey(preHour); - if (records.size() != 0) { - records.forEach(x -> { - x.setBizType(1); - x.setCreateTime(nowTime); - x.setDays(day); - x.setHours(hour); - x.setUuid(1 + "_" + x.getKeyName() + "_" + hour); - }); - } - log.info("每小时统计最热点,时间：{}, 行数：{}", now.toString(), records.size()); - List statistics = keyRecordMapper.statisticsByRule(preHour); - if (statistics.size() != 0) { - statistics.forEach(x -> { - x.setBizType(6); - x.setRule(x.getRule()); - x.setCreateTime(nowTime); - x.setDays(day); - x.setHours(hour); - x.setUuid(6 + "_" + x.getKeyName() + "_" + hour); - }); - log.info("每小时统计规则,时间：{}, data list：{}", now.toString(), JSON.toJSONString(statistics)); - records.addAll(statistics); - } - if (records.size() > 0) { - int row = statisticsMapper.batchInsert(records); + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 + scheduledExecutorService.scheduleAtFixedRate(() -> { + try { + LocalDateTime now = LocalDateTime.now(); + Date nowTime = DateUtils.ldtToDate(now); + int day = DateUtils.nowDay(now); + int hour = DateUtils.nowHour(now); + SearchReq preHour = new SearchReq(now.minusHours(1)); + List records = keyRecordMapper.maxHotKey(preHour); + if (records.size() != 0) { + records.forEach(x -> { + x.setBizType(1); + x.setCreateTime(nowTime); + x.setDays(day); + x.setHours(hour); + x.setUuid(1 + "_" + x.getKeyName() + "_" + hour); + }); + } + log.info("每小时统计最热点,时间：{}, 行数：{}", now.toString(), records.size()); + List statistics = keyRecordMapper.statisticsByRule(preHour); + if (statistics.size() != 0) { + statistics.forEach(x -> { + x.setBizType(6); + x.setRule(x.getRule()); + x.setCreateTime(nowTime); + x.setDays(day); + x.setHours(hour); + x.setUuid(6 + "_" + x.getKeyName() + "_" + hour); + }); + log.info("每小时统计规则,时间：{}, data list：{}", now.toString(), JSON.toJSONString(statistics)); + records.addAll(statistics); + } + if (records.size() > 0) { + int row = statisticsMapper.batchInsert(records); + } + } catch (Exception e) { + e.printStackTrace(); } - } catch (Exception e) { - e.printStackTrace(); - } - + }, 2, 1, TimeUnit.HOURS); } - /** * 每分钟统计一次record 表 结果记录到统计表 */ - @Scheduled(cron = "0 */1 * * * ?") + @PostConstruct public void offlineStatisticsRule() { - try { - LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtils.ldtToDate(now); - int day = DateUtils.nowDay(now); - int hour = DateUtils.nowHour(now); - int minus = DateUtils.nowMinus(now); - - List records = keyRecordMapper.statisticsByRule(new SearchReq(now.minusMinutes(1))); - if (records.size() == 0) { - return; - } - records.forEach(x -> { - x.setBizType(5); - x.setRule(x.getRule()); - x.setCreateTime(nowTime); - x.setDays(day); - x.setHours(hour); - x.setMinutes(minus); - // 骚操作 临时解决没有rule字段的问题 - x.setUuid(5 + "_" + x.getKeyName() + "_" + minus); - }); - int row = statisticsMapper.batchInsert(records); + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 + scheduledExecutorService.scheduleAtFixedRate(() -> { + try { + LocalDateTime now = LocalDateTime.now(); + Date nowTime = DateUtils.ldtToDate(now); + int day = DateUtils.nowDay(now); + int hour = DateUtils.nowHour(now); + int minus = DateUtils.nowMinus(now); + + List records = keyRecordMapper.statisticsByRule(new SearchReq(now.minusMinutes(1))); + if (records.size() == 0) { + return; + } + records.forEach(x -> { + x.setBizType(5); + x.setRule(x.getRule()); + x.setCreateTime(nowTime); + x.setDays(day); + x.setHours(hour); + x.setMinutes(minus); + // 骚操作 临时解决没有rule字段的问题 + x.setUuid(5 + "_" + x.getKeyName() + "_" + minus); + }); + int row = statisticsMapper.batchInsert(records); // log.info("每分钟统计规则，时间：{}, 影响行数：{}, data list:{}", now.toString(), row, JSON.toJSONString(records)); - } catch (Exception e) { - e.printStackTrace(); - } + } catch (Exception e) { + e.printStackTrace(); + } + }, 2, 1, TimeUnit.MINUTES); } @@ -217,28 +222,30 @@ public class DataHandler { /** * 每天根据app的配置清理过期数据 */ -// @Scheduled(cron = "0 0 3 * * ?") - @Scheduled(fixedRate = 5000) //每隔5秒删1万 + @PostConstruct public void clearExpireData() { - List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); - for (KeyValue kv : keyValues) { - try { - String val = kv.getValue().toStringUtf8(); - AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); - String app = cfg.getApp(); - //保存几天 - Integer days = cfg.getDataTtl(); - DateTime dateTime = DateUtil.offsetDay(new Date(), -days); - - - summaryMapper.clearExpireData(app, dateTime); - keyRecordMapper.clearExpireData(app, dateTime); - statisticsMapper.clearExpireData(app, dateTime); - } catch (Exception e) { - e.printStackTrace(); + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 + scheduledExecutorService.scheduleAtFixedRate(() -> { + List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + for (KeyValue kv : keyValues) { + try { + String val = kv.getValue().toStringUtf8(); + AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); + String app = cfg.getApp(); + //保存几天 + Integer days = cfg.getDataTtl(); + DateTime dateTime = DateUtil.offsetDay(new Date(), -days); + + + summaryMapper.clearExpireData(app, dateTime); + keyRecordMapper.clearExpireData(app, dateTime); + statisticsMapper.clearExpireData(app, dateTime); + } catch (Exception e) { + e.printStackTrace(); + } } - } - + }, 1, 5, TimeUnit.MINUTES); } -- Gitee From 7761839c406f3c9b9c03a9bac5e7d8d2ff2f03b4 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 4 Nov 2020 14:58:46 +0800 Subject: [PATCH 46/59] =?UTF-8?q?=E5=A4=A7=E7=9B=98=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/controller/KeyController.java | 2 +- .../biz/service/impl/KeyServiceImpl.java | 2 +- .../dashboard/common/monitor/DataHandler.java | 104 +++++++++--------- .../dashboard/netty/HotKeyReceiver.java | 2 +- .../hotkey/dashboard/warn/DongDongUtil.java | 2 +- .../static/admin/assets/css/extra-pages.css | 2 +- 6 files changed, 57 insertions(+), 57 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java index 2c0b36f..fab681b 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java @@ -124,7 +124,7 @@ public class KeyController extends BaseController { /*@GetMapping("/edit/{id}") public String edit(@PathVariable("id") Long id, ModelMap modelMap){ - modelMap.put("key", keyService.selectByPk(id)); + modelMap.writeToLocalCaffeine("key", keyService.selectByPk(id)); return prefix + "/edit"; }*/ diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java index c7a5f0a..83aacb0 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java @@ -176,7 +176,7 @@ public class KeyServiceImpl implements KeyService { hotKeyModel.setCreateTime(System.currentTimeMillis()); hotKeyModel.setAppName(key.getAppName()); hotKeyModel.setKey(key.getKey()); - HotKeyReceiver.put(hotKeyModel); + HotKeyReceiver.writeToLocalCaffeine(hotKeyModel); return logMapper.insertSelective(new ChangeLog(key.getAppName(), Constant.HOTKEY_CHANGE, "", key.getKey(), key.getUpdater(), SystemClock.now() + "")); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 2f49b82..be7d7a1 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -63,16 +63,8 @@ public class DataHandler { /** - * 入队 + * 开始定时将keyRecord入库 */ - public void offer(IRecord record) { - try { - RECORD_QUEUE.put(record); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - public void insertRecords() { while (true) { try { @@ -98,6 +90,57 @@ public class DataHandler { } + /** + * 读取netty发来的热key，进行初步处理，并写入本地caffeine + */ + public void dealHotKey() { + while (true) { + try { + HotKeyModel model = HotKeyReceiver.take(); + //将该key放入实时热key本地缓存中 + if (model != null) { + //将key放到队列里，供入库时分批调用 + putRecord(model.getAppName(), model.getKey(), model.getCreateTime()); + //获取发来的这个热key，存入本地caffeine，设置过期时间 + HotKeyReceiver.writeToLocalCaffeine(model); + } + } catch (Exception e) { + e.printStackTrace(); + } + } + } + + /** + * 将待入库的热key放到队列 + */ + private void putRecord(String app, String key, long createTime) { + try { + RECORD_QUEUE.put(new IRecord() { + @Override + public String appNameKey() { + return app + "/" + key; + } + + @Override + public String value() { + return UUID.randomUUID().toString(); + } + + @Override + public int type() { + return 0; + } + + @Override + public Date createTime() { + return new Date(createTime); + } + }); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + /** * 处理热点key和记录 @@ -272,49 +315,6 @@ public class DataHandler { } - - public void dealHotKey() { - while (true) { - try { - //获取发来的这个热key，存入本地caffeine，设置过期时间 - HotKeyModel model = HotKeyReceiver.take(); - //将该key放入实时热key本地缓存中 - if (model != null) { - HotKeyReceiver.put(model); - putRecord(model.getAppName(), model.getKey()); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - - private void putRecord(String app, String key) { - this.offer(new IRecord() { - @Override - public String appNameKey() { - return app + "/" + key; - } - - @Override - public String value() { - return UUID.randomUUID().toString(); - } - - @Override - public int type() { - return 0; - } - - @Override - public Date createTime() { - return new Date(); - } - }); - } - - /** * 查询条数 比对配置 发送报警 */ diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java index 88057c9..5f5c5f2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/netty/HotKeyReceiver.java @@ -56,7 +56,7 @@ public class HotKeyReceiver { /** * 将热key存入本地缓存，设置过期时间 */ - public static void put(HotKeyModel hotKeyModel) { + public static void writeToLocalCaffeine(HotKeyModel hotKeyModel) { String appNameKey = hotKeyModel.getAppName() + "/" + hotKeyModel.getKey(); KeyRule keyRule = RuleUtil.findByKey(appNameKey); if (keyRule == null) { diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java index e3f635e..7ec7d85 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java @@ -99,7 +99,7 @@ public class DongDongUtil { JSONObject extend = new JSONObject(); extend.put("url", "http://hotkey.jd.com"); - /* extend.put("pic", "https://img14.360buyimg.com/imagetools/jfs/t1/144156/7/8194/7263/5f5b34baEdfd90a49/a15144ab447bfbf2.png");*/ + /* extend.writeToLocalCaffeine("pic", "https://img14.360buyimg.com/imagetools/jfs/t1/144156/7/8194/7263/5f5b34baEdfd90a49/a15144ab447bfbf2.png");*/ json.put("extend", extend); params.add(new BasicNameValuePair("jsonMsg",JSON.toJSONString(json))); diff --git a/dashboard/src/main/resources/static/admin/assets/css/extra-pages.css b/dashboard/src/main/resources/static/admin/assets/css/extra-pages.css index 54ca0fc..c067452 100644 --- a/dashboard/src/main/resources/static/admin/assets/css/extra-pages.css +++ b/dashboard/src/main/resources/static/admin/assets/css/extra-pages.css @@ -333,7 +333,7 @@ h2.headline.text-info { .lockscreen > body { background: transparent; } -/* We will put the dynamically generated digital clock here */ +/* We will writeToLocalCaffeine the dynamically generated digital clock here */ .lockscreen .headline { color: #fff; text-shadow: 1px 3px 5px rgba(0, 0, 0, 0.5); -- Gitee From 004a146e7ad285236a9080e6fde55a2a20926c10 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Wed, 4 Nov 2020 15:39:45 +0800 Subject: [PATCH 47/59] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E8=AE=B0=E5=BD=95?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/controller/KeyController.java | 41 +++++++++++++++---- .../dashboard/biz/service/KeyService.java | 2 + .../biz/service/impl/KeyServiceImpl.java | 5 +++ .../hotkey/dashboard/util/DateUtils.java | 5 +++ .../hotkey/dashboard/util/ExcelUtil.java | 3 ++ .../resources/templates/admin/key/list.html | 2 +- 6 files changed, 50 insertions(+), 8 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java index fab681b..e81be35 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java @@ -122,13 +122,6 @@ public class KeyController extends BaseController { } - /*@GetMapping("/edit/{id}") - public String edit(@PathVariable("id") Long id, ModelMap modelMap){ - modelMap.writeToLocalCaffeine("key", keyService.selectByPk(id)); - return prefix + "/edit"; - }*/ - - @PostMapping("/edit") @ResponseBody public Result editSave(KeyTimely key) { @@ -157,6 +150,28 @@ public class KeyController extends BaseController { } + @RequestMapping(value = "/exportList", method = RequestMethod.GET) + @ResponseBody + public void exportList(HttpServletResponse resp,String startTime,String endTime,String app,String key){ + SearchReq req = new SearchReq(); + if(StringUtil.isNotEmpty(startTime)){ + req.setStartTime(DateUtils.strToDate(startTime)); + } + if(StringUtil.isNotEmpty(endTime)){ + req.setEndTime(DateUtils.strToDate(endTime)); + } + req.setApp(app); + req.setKey(key); + List records = keyService.listKeyRecord(req); + if(records.size() > ExcelUtil.MAX_ROW){ + records = records.subList(0,ExcelUtil.MAX_ROW); + } + List> rows = transformList(records); + ExcelDataDto data = new ExcelDataDto("keyRecord.xlsx", Constant.HEAD,rows); + ExcelUtil.exportExcel(resp,data); + } + + private List> transform(List records){ List> rows = new ArrayList<>(); @@ -169,5 +184,17 @@ public class KeyController extends BaseController { } return rows; } + + private List> transformList(List records){ + List> rows = new ArrayList<>(); + for (KeyRecord record : records) { + List list = new ArrayList<>(); + list.add(record.getKey()); + list.add(record.getAppName()); + list.add(DateUtils.dateToStr(record.getCreateTime())); + rows.add(list); + } + return rows; + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java index d8908f9..2312797 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/KeyService.java @@ -22,6 +22,8 @@ import java.util.List; public interface KeyService { + List listKeyRecord(SearchReq param); + PageInfo pageKeyRecord(PageReq page, SearchReq param); int insertKeyByUser(KeyTimely keyTimely); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java index 83aacb0..fddcaa4 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java @@ -154,6 +154,11 @@ public class KeyServiceImpl implements KeyService { } + @Override + public List listKeyRecord(SearchReq param) { + return recordMapper.listKeyRecord(param); + } + @Override public PageInfo pageKeyRecord(PageReq page, SearchReq param) { PageHelper.startPage(page.getPageNum(), page.getPageSize()); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java index a854aba..c5c8c53 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java @@ -40,6 +40,11 @@ public class DateUtils { return null; } + public static String dateToStr(Date date){ + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + return format.format(date); + } + public static LocalDateTime strToLdt(String str, String pattern){ return LocalDateTime.parse(str, DateTimeFormatter.ofPattern(pattern)); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/ExcelUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/ExcelUtil.java index 7641f0e..fa13456 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/ExcelUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/ExcelUtil.java @@ -13,6 +13,9 @@ import org.slf4j.LoggerFactory; public class ExcelUtil { + + public static final int MAX_ROW = 60000; + private static Logger log = LoggerFactory.getLogger(ExcelUtil.class); public static void exportExcel(HttpServletResponse response, ExcelDataDto data) { diff --git a/dashboard/src/main/resources/templates/admin/key/list.html b/dashboard/src/main/resources/templates/admin/key/list.html index 6f755c8..664af98 100644 --- a/dashboard/src/main/resources/templates/admin/key/list.html +++ b/dashboard/src/main/resources/templates/admin/key/list.html @@ -231,7 +231,7 @@ $(function(){ }) function exportHot() { - $("#keyForm").attr("action","/key/export");//改变表单的提交地址为下载的地址 + $("#keyForm").attr("action","/key/exportList");//改变表单的提交地址为下载的地址 $("#keyForm").submit();//提交表单 } -- Gitee From 3fb31cf63d03125ef9457f74cc30bcab60312a63 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Wed, 4 Nov 2020 16:07:14 +0800 Subject: [PATCH 48/59] =?UTF-8?q?=E6=89=BE=E5=9B=9E=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=A1=A8=E5=A4=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/biz/controller/KeyController.java | 2 +- .../hotkey/dashboard/common/domain/Constant.java | 4 ++++ .../hotkey/dashboard/common/monitor/DataHandler.java | 10 +++++++++- 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java index e81be35..7ccab54 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java @@ -167,7 +167,7 @@ public class KeyController extends BaseController { records = records.subList(0,ExcelUtil.MAX_ROW); } List> rows = transformList(records); - ExcelDataDto data = new ExcelDataDto("keyRecord.xlsx", Constant.HEAD,rows); + ExcelDataDto data = new ExcelDataDto("keyRecord.xlsx", Constant.RECORD_HEAD,rows); ExcelUtil.exportExcel(resp,data); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java index 30dca53..6847429 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java @@ -66,6 +66,10 @@ public class Constant { static { HEAD.add("热点key"); HEAD.add("次数"); HEAD.add("所属APP");} + public static final List RECORD_HEAD = new ArrayList<>(); + + static { RECORD_HEAD.add("热点key"); RECORD_HEAD.add("所属APP"); RECORD_HEAD.add(" 时间");} + public static final int WARN_INIT_MIN = -1; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index be7d7a1..537be41 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -63,8 +63,16 @@ public class DataHandler { /** - * 开始定时将keyRecord入库 + * 入队 */ + public void offer(IRecord record) { + try { + RECORD_QUEUE.put(record); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + public void insertRecords() { while (true) { try { -- Gitee From 28cf3d298c51a56028213fc7e862a7a098473688 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 24 Nov 2020 14:41:42 +0800 Subject: [PATCH 49/59] =?UTF-8?q?=E6=89=B9=E9=87=8F=E6=8E=A8=E9=80=81?= =?UTF-8?q?=E7=83=ADkey=E5=88=B0=E5=AE=A2=E6=88=B7=E7=AB=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/netty/NettyClientHandler.java | 5 +- .../worker/netty/pusher/AppServerPusher.java | 76 +++++++++++++++---- .../worker/netty/pusher/DashboardPusher.java | 40 +++++++++- .../worker/netty/pusher/KeyUploader.java | 46 ----------- .../netty/pusher/store/HotkeyTempStore.java | 27 ------- 5 files changed, 103 insertions(+), 91 deletions(-) delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClientHandler.java b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClientHandler.java index c16ebf2..6710017 100755 --- a/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClientHandler.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/netty/NettyClientHandler.java @@ -65,8 +65,9 @@ public class NettyClientHandler extends SimpleChannelInboundHandler { if (CollectionUtil.isEmpty(msg.getHotKeyModels())) { return; } - HotKeyModel model = msg.getHotKeyModels().get(0); - EventBusCenter.getInstance().post(new ReceiveNewKeyEvent(model)); + for (HotKeyModel model : msg.getHotKeyModels()) { + EventBusCenter.getInstance().post(new ReceiveNewKeyEvent(model)); + } } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java index 5a2e150..65a3c5e 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/AppServerPusher.java @@ -1,44 +1,92 @@ package com.jd.platform.hotkey.worker.netty.pusher; +import cn.hutool.core.collection.CollectionUtil; +import com.google.common.collect.Queues; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.worker.model.AppInfo; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; +import com.jd.platform.hotkey.worker.tool.AsyncPool; import org.springframework.stereotype.Component; -import java.util.Arrays; +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; /** * 推送到各客户端服务器 + * * @author wuweifeng wrote on 2020-02-24 * @version 1.0 */ @Component public class AppServerPusher implements IPusher { + /** + * 热key集中营 + */ + private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); /** * 给客户端推key信息 */ @Override public void push(HotKeyModel model) { - for (AppInfo appInfo : ClientInfoHolder.apps) { - if (model.getAppName().equals(appInfo.getAppName())) { - - HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.RESPONSE_NEW_KEY); - hotKeyMsg.setHotKeyModels(Arrays.asList(model)); - - //整个app全部发送 - appInfo.groupPush(hotKeyMsg); - - return; - } - } - + hotKeyStoreQueue.offer(model); } @Override public void remove(HotKeyModel model) { push(model); } + + /** + * 和dashboard那边的推送主要区别在于，给app推送每10ms一次，dashboard那边1s一次 + */ + @PostConstruct + public void batchPushToClient() { + AsyncPool.asyncDo(() -> { + while (true) { + try { + List tempModels = new ArrayList<>(); + //每10ms推送一次 + Queues.drain(hotKeyStoreQueue, tempModels, 10, 10, TimeUnit.MILLISECONDS); + if (CollectionUtil.isEmpty(tempModels)) { + continue; + } + + Map> allAppHotKeyModels = new HashMap<>(); + + //拆分出每个app的热key集合，按app分堆 + for (HotKeyModel hotKeyModel : tempModels) { + List oneAppModels = allAppHotKeyModels.computeIfAbsent(hotKeyModel.getAppName(), (key) -> new ArrayList<>()); + oneAppModels.add(hotKeyModel); + } + + //遍历所有app，进行推送 + for (AppInfo appInfo : ClientInfoHolder.apps) { + List list = allAppHotKeyModels.get(appInfo.getAppName()); + if (CollectionUtil.isEmpty(list)) { + continue; + } + + HotKeyMsg hotKeyMsg = new HotKeyMsg(MessageType.RESPONSE_NEW_KEY); + hotKeyMsg.setHotKeyModels(list); + + //整个app全部发送 + appInfo.groupPush(hotKeyMsg); + } + + allAppHotKeyModels = null; + + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java index aa420cd..09678dd 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/DashboardPusher.java @@ -1,9 +1,19 @@ package com.jd.platform.hotkey.worker.netty.pusher; +import cn.hutool.core.collection.CollectionUtil; +import com.google.common.collect.Queues; import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.worker.netty.pusher.store.HotkeyTempStore; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.worker.netty.dashboard.DashboardHolder; +import com.jd.platform.hotkey.worker.tool.AsyncPool; import org.springframework.stereotype.Component; +import javax.annotation.PostConstruct; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + /** * 将热key推送到dashboard供入库 * @author wuweifeng @@ -12,13 +22,39 @@ import org.springframework.stereotype.Component; */ @Component public class DashboardPusher implements IPusher { + /** + * 热key集中营 + */ + private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); + @Override public void push(HotKeyModel model) { - HotkeyTempStore.push(model); + hotKeyStoreQueue.offer(model); } @Override public void remove(HotKeyModel model) { } + + @PostConstruct + public void uploadToDashboard() { + AsyncPool.asyncDo(() -> { + while (true) { + try { + //要么key达到1千个，要么达到1秒，就汇总上报给etcd一次 + List tempModels = new ArrayList<>(); + Queues.drain(hotKeyStoreQueue, tempModels, 1000, 1, TimeUnit.SECONDS); + if (CollectionUtil.isEmpty(tempModels)) { + continue; + } + + //将热key推到dashboard + DashboardHolder.flushToDashboard(FastJsonUtils.convertObjectToJSON(tempModels)); + } catch (Exception e) { + e.printStackTrace(); + } + } + }); + } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java deleted file mode 100644 index 8ee6e11..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/KeyUploader.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher; - -import cn.hutool.core.collection.CollectionUtil; -import com.google.common.collect.Queues; -import com.jd.platform.hotkey.common.model.HotKeyModel; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import com.jd.platform.hotkey.worker.netty.dashboard.DashboardHolder; -import com.jd.platform.hotkey.worker.netty.pusher.store.HotkeyTempStore; -import com.jd.platform.hotkey.worker.tool.AsyncPool; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; - -/** - * 定时批量往dashboard发送热key，供入库 - * @author wuweifeng - * @version 1.0 - * @date 2020-08-31 - */ -@Component -public class KeyUploader { - - @PostConstruct - public void uploadToDashboard() { - AsyncPool.asyncDo(() -> { - while (true) { - try { - //要么key达到1千个，要么达到1秒，就汇总上报给etcd一次 - List tempModels = new ArrayList<>(); - Queues.drain(HotkeyTempStore.getQueue(), tempModels, 1000, 1, TimeUnit.SECONDS); - if (CollectionUtil.isEmpty(tempModels)) { - continue; - } - - //将热key推到dashboard - DashboardHolder.flushToDashboard(FastJsonUtils.convertObjectToJSON(tempModels)); - } catch (Exception e) { - e.printStackTrace(); - } - } - }); - } -} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java deleted file mode 100644 index c93d18d..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/pusher/store/HotkeyTempStore.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.jd.platform.hotkey.worker.netty.pusher.store; - -import com.jd.platform.hotkey.common.model.HotKeyModel; - -import java.util.concurrent.LinkedBlockingQueue; - -/** - * 已热待上报入库的热key集中营 - * @author wuweifeng - * @version 1.0 - * @date 2020-08-31 - */ -public class HotkeyTempStore { - /** - * 热key集中营 - */ - private static LinkedBlockingQueue hotKeyStoreQueue = new LinkedBlockingQueue<>(); - - public static void push(HotKeyModel model) { - hotKeyStoreQueue.offer(model); - } - - public static LinkedBlockingQueue getQueue() { - return hotKeyStoreQueue; - } - -} -- Gitee From 8d4847c92447b3f38a9c88cfa77e32f1255d6a81 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 22 Dec 2020 14:49:34 +0800 Subject: [PATCH 50/59] =?UTF-8?q?=E6=89=B9=E9=87=8F=E6=8E=A8=E9=80=81?= =?UTF-8?q?=E7=83=ADkey=E5=88=B0=E5=AE=A2=E6=88=B7=E7=AB=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/biz/service/impl/RuleServiceImpl.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java index 34732a3..76a4004 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java @@ -6,6 +6,7 @@ import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; +import com.google.protobuf.ByteString; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; @@ -125,7 +126,11 @@ public class RuleServiceImpl implements RuleService { String app = rules.getApp(); KeyValue kv = configCenter.getKv(ConfigConstant.rulePath + app); - String from = kv.getValue().toStringUtf8(); + String from = null; + ByteString string = kv.getValue(); + if (string != null) { + from = kv.getValue().toStringUtf8(); + } String to = JSON.toJSONString(rules); configCenter.put(ConfigConstant.rulePath + app, rules.getRules()); -- Gitee From c57a9acd36ebbfb529a00e862270ea11aa5d9037 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 22 Dec 2020 14:58:03 +0800 Subject: [PATCH 51/59] =?UTF-8?q?=E5=88=A0=E9=99=A4=E8=A7=84=E5=88=99?= =?UTF-8?q?=E5=90=8E=E5=86=8D=E6=B7=BB=E5=8A=A0=E8=A7=84=E5=88=99=E6=8A=A5?= =?UTF-8?q?=E7=A9=BA=E6=8C=87=E9=92=88=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../biz/service/impl/RuleServiceImpl.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java index 76a4004..23b225c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/RuleServiceImpl.java @@ -6,18 +6,19 @@ import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; -import com.google.protobuf.ByteString; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; -import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; import com.jd.platform.hotkey.dashboard.biz.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.biz.mapper.RulesMapper; import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; -import com.jd.platform.hotkey.dashboard.model.*; import com.jd.platform.hotkey.dashboard.biz.service.RuleService; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; +import com.jd.platform.hotkey.dashboard.model.ChangeLog; +import com.jd.platform.hotkey.dashboard.model.Rule; +import com.jd.platform.hotkey.dashboard.model.Rules; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StringUtils; @@ -127,8 +128,7 @@ public class RuleServiceImpl implements RuleService { KeyValue kv = configCenter.getKv(ConfigConstant.rulePath + app); String from = null; - ByteString string = kv.getValue(); - if (string != null) { + if (kv != null) { from = kv.getValue().toStringUtf8(); } String to = JSON.toJSONString(rules); -- Gitee From bbf7e3d10c9733a658a59df14dd3b011e1abd3e9 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 22 Dec 2020 18:29:29 +0800 Subject: [PATCH 52/59] =?UTF-8?q?=E5=85=A8=E5=B1=80=E6=9C=AA=E7=9F=A5?= =?UTF-8?q?=E5=BC=82=E5=B8=B8=E7=BD=AE=E5=85=A5error=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/dashboard/common/config/MyExceptionHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java index 401f52c..a052472 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java @@ -33,8 +33,9 @@ public class MyExceptionHandler { @ExceptionHandler(value =Exception.class) @ResponseBody - public Result exceptionHandler(Exception e){ + public Result exceptionHandler(Exception e,HttpServletResponse resp){ logger.error("未知异常：",e); + resp.setStatus(500); return Result.error(ResultEnum.BIZ_ERROR); } } -- Gitee From 414ca736a90187ce5d20b22d0dcb73c19e3154a4 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 25 May 2021 11:21:09 +0800 Subject: [PATCH 53/59] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=BC=BA=E5=88=B6?= =?UTF-8?q?=E6=96=B0=E5=A2=9Ekey-value=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/client/callback/JdHotKeyStore.java | 10 ++++++++++ .../client/core/key/TurnCountCollector.java | 2 +- .../hotkey/common/coder/MsgDecoder.java | 18 ------------------ 3 files changed, 11 insertions(+), 19 deletions(-) diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java index 17e4fc2..3647ff8 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java @@ -83,6 +83,16 @@ public class JdHotKeyStore { } } + /** + * 强制给value赋值 + */ + public static void forceSet(String key, Object value) { + ValueModel valueModel = ValueModel.defaultValue(key); + if (valueModel != null) { + valueModel.setValue(value); + } + } + /** * 获取value，如果value不存在则发往netty */ diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java index 329ec7c..7e4f146 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java @@ -83,7 +83,7 @@ public class TurnCountCollector implements IKeyCollector map) { + private void put(String key, boolean isHot, ConcurrentHashMap map) { //如key是pin_的前缀，则存储pin_ String rule = KeyRuleHolder.rule(key); //不在规则内的不处理 diff --git a/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgDecoder.java b/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgDecoder.java index 0710ca1..492c5e7 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgDecoder.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/coder/MsgDecoder.java @@ -23,24 +23,6 @@ public class MsgDecoder extends ByteToMessageDecoder { list.add(ProtostuffUtils.deserialize(body, HotKeyMsg.class)); -// if (in.readableBytes() < 4) { -// return; -// } -// in.markReaderIndex(); -// int dataLength = in.readInt(); -// if (dataLength < 0) { -// channelHandlerContext.close(); -// } -// if (in.readableBytes() < dataLength) { -// in.resetReaderIndex(); -// return; -// } -// -// byte[] data = new byte[dataLength]; -// in.readBytes(data); -// -// Object obj = ProtostuffUtils.deserialize(data, HotKeyMsg.class); -// list.add(obj); } catch (Exception e) { e.printStackTrace(); } -- Gitee From 81caf1bab0ede93b3997e8458e1ec3408c9890f4 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 25 May 2021 18:28:54 +0800 Subject: [PATCH 54/59] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=BC=BA=E5=88=B6?= =?UTF-8?q?=E6=96=B0=E5=A2=9Ekey-value=E6=96=B9=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/jd/platform/hotkey/client/callback/JdHotKeyStore.java | 1 + 1 file changed, 1 insertion(+) diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java index 3647ff8..356fafa 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java @@ -91,6 +91,7 @@ public class JdHotKeyStore { if (valueModel != null) { valueModel.setValue(value); } + setValueDirectly(key, valueModel); } /** -- Gitee From d35be9e8a285ad84128cca19363f24cfd9b8da96 Mon Sep 17 00:00:00 2001 From: oawang <349339884@qq.com> Date: Tue, 29 Jun 2021 17:59:58 +0800 Subject: [PATCH 55/59] meger v0.04 --- .../hotkey/client/callback/ValueModel.java | 4 + .../client/core/key/NettyKeyPusher.java | 1 - .../client/core/key/TurnCountCollector.java | 52 ++++- .../client/core/key/TurnKeyCollector.java | 2 + .../common/configcenter/ConfigConstant.java | 8 + .../platform/hotkey/common/tool/Constant.java | 2 +- dashboard/pom.xml | 15 -- .../biz/controller/KeyController.java | 12 +- .../biz/service/impl/KeyServiceImpl.java | 22 +- .../dashboard/common/domain/req/ChartReq.java | 6 +- .../common/domain/req/SearchReq.java | 4 +- .../dashboard/common/monitor/DataHandler.java | 199 ++++++++---------- .../dashboard/common/monitor/PushHandler.java | 89 -------- .../dashboard/erp/ErpConfiguration.java | 24 --- .../hotkey/dashboard/erp/ErpProperties.java | 75 ------- .../dashboard/erp/ErpUimInterceptor.java | 56 ----- .../hotkey/dashboard/etcd/EtcdMonitor.java | 55 ++++- .../dashboard/interceptor/JwtInterceptor.java | 6 +- .../interceptor/MyWebAppConfigurer.java | 8 +- .../hotkey/dashboard/util/CommonUtil.java | 72 ++++++- .../util/{DateUtils.java => DateUtil.java} | 2 +- .../dashboard/warn/DongDongApiManager.java | 60 ------ .../hotkey/dashboard/warn/DongDongToken.java | 10 - .../hotkey/dashboard/warn/DongDongUtil.java | 119 ----------- dashboard/src/main/resources/application.yml | 11 +- .../dashboard/biz/mapper/KeyRecordMapper.xml | 12 +- .../src/main/resources/logback-spring.xml | 2 +- .../static/admin/assets/css/extra-pages.css | 2 +- .../static/admin/common/js/key-add.js | 4 + .../static/admin/common/js/key-edit.js | 4 + .../static/admin/common/js/rule-add.js | 4 + .../static/admin/common/js/rule-edit.js | 8 +- .../static/admin/common/js/user-add.js | 4 + .../static/admin/common/js/user-edit.js | 4 + .../templates/admin/common/html/footer.html | 4 +- .../templates/admin/common/html/leftMenu.html | 4 +- .../templates/admin/common/html/title.html | 4 +- .../templates/admin/common/html/topMenu.html | 22 +- .../main/resources/templates/admin/index.html | 15 +- .../resources/templates/admin/key/list.html | 2 +- .../main/resources/templates/admin/main.html | 8 +- .../src/main/resources/templates/login.html | 1 - .../java/com/jd/platform/sample/Starter.java | 56 +---- .../sample/controller/TestController.java | 13 +- sample/src/main/resources/application.yml | 4 - sample/src/main/resources/logback-spring.xml | 2 +- worker/pom.xml | 6 - .../jd/platform/hotkey/worker/config/Ump.java | 6 +- .../hotkey/worker/starters/EtcdStarter.java | 2 +- .../worker/tool/DirectMemoryReporter.java | 40 ---- .../hotkey/worker/tool/SlidingWindow.java | 6 - worker/src/main/resources/application.yml | 1 + worker/src/main/resources/logback-spring.xml | 2 +- 53 files changed, 382 insertions(+), 774 deletions(-) delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpConfiguration.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpProperties.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/{DateUtils.java => DateUtil.java} (99%) delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongToken.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/tool/DirectMemoryReporter.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/ValueModel.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/ValueModel.java index 0265cbc..e5ff17e 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/ValueModel.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/ValueModel.java @@ -40,6 +40,10 @@ public class ValueModel { return createTime; } + public void setCreateTime(long createTime) { + this.createTime = createTime; + } + public int getDuration() { return duration; } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java index 988c3cc..f6210f9 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java @@ -46,7 +46,6 @@ public class NettyKeyPusher implements IKeyPusher { hotKeyMsg.setHotKeyModels(batch); channel.writeAndFlush(hotKeyMsg).sync(); } catch (Exception e) { - e.printStackTrace(); try { InetSocketAddress insocket = (InetSocketAddress) channel.remoteAddress(); JdLogger.error(getClass(),"flush error " + insocket.getAddress().getHostAddress()); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java index 7e4f146..d227491 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java @@ -11,8 +11,9 @@ import java.util.Date; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; +import java.util.stream.Collectors; /** * 热点数量统计 @@ -36,6 +37,8 @@ public class TurnCountCollector implements IKeyCollector lockAndGetResult() { //自增后，对应的map就会停止被写入，等待被读取 @@ -56,14 +59,45 @@ public class TurnCountCollector implements IKeyCollector get(ConcurrentHashMap map) { - List list = new ArrayList<>(); + //根据待转换并上报的统计数据的数据量选择是否启用并行参数转换 + if (map.size()>DATA_CONVERT_SWITCH_THRESHOLD){ + return parallelConvert(map); + }else { + return syncConvert(map); + } + } + + /** + * 在数据量足够大的情况下 并行转换可以拥有比串行for循环更好的性能 + * @param map 统计数据 + * @return 待上报数据 + */ + private List parallelConvert(ConcurrentHashMap map) { + return map.entrySet().parallelStream().map(entry->{ + String key = entry.getKey(); + HitCount hitCount = entry.getValue(); + KeyCountModel keyCountModel = new KeyCountModel(); + keyCountModel.setTotalHitCount((int)hitCount.totalHitCount.sum()); + keyCountModel.setRuleKey(key); + keyCountModel.setHotHitCount((int)hitCount.hotHitCount.sum()); + return keyCountModel; + }).collect(Collectors.toList()); + } + + /** + * 在数据量不大的情况下,使用同步for循环进行数据转换性能也不错 + * @param map 统计数据 + * @return 待上报数据 + */ + private List syncConvert(ConcurrentHashMap map) { + List list = new ArrayList<>(map.size()); for (Map.Entry entry : map.entrySet()) { String key = entry.getKey(); HitCount hitCount = entry.getValue(); KeyCountModel keyCountModel = new KeyCountModel(); - keyCountModel.setTotalHitCount(hitCount.totalHitCount.get()); + keyCountModel.setTotalHitCount((int)hitCount.totalHitCount.sum()); keyCountModel.setRuleKey(key); - keyCountModel.setHotHitCount(hitCount.hotHitCount.get()); + keyCountModel.setHotHitCount((int)hitCount.hotHitCount.sum()); list.add(keyCountModel); } return list; @@ -83,7 +117,7 @@ public class TurnCountCollector implements IKeyCollector map) { + public void put(String key, boolean isHot, ConcurrentHashMap map) { //如key是pin_的前缀，则存储pin_ String rule = KeyRuleHolder.rule(key); //不在规则内的不处理 @@ -97,9 +131,9 @@ public class TurnCountCollector implements IKeyCollector new HitCount()); if (isHot) { - hitCount.hotHitCount.incrementAndGet(); + hitCount.hotHitCount.increment(); } - hitCount.totalHitCount.incrementAndGet(); + hitCount.totalHitCount.increment(); } private String nowTime() { @@ -109,7 +143,7 @@ public class TurnCountCollector implements IKeyCollector } if (atomicLong.get() % 2 == 0) { //不存在时返回null并将key-value放入，已有相同key时，返回该key对应的value，并且不覆盖 + HotKeyModel model = map0.putIfAbsent(key, hotKeyModel); if (model != null) { model.setCount(model.getCount() + hotKeyModel.getCount()); diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index 56c6610..009a107 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -17,6 +17,10 @@ public interface ConfigConstant { * dashboard的ip存这里 */ String dashboardPath = "/jd/dashboard/"; + /** + * 该app所有的workers地址的path。需要手工分配，默认每个app都用所有的worker + */ + String appWorkerPath = null; /** * 所有的客户端规则（譬如哪个app的哪些前缀的才参与计算） */ @@ -45,6 +49,10 @@ public interface ConfigConstant { * totalReceiveKeyCount该worker接收到的key总量，每10秒上报一次 */ String totalReceiveKeyCount = "/jd/totalKeyCount/"; + /** + * bufferPool直接内存 + */ + String bufferPoolPath = "/jd/bufferPool/"; /** * 存放客户端hotKey访问次数和总访问次数的path diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java index 81a40ec..1d3ed6b 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java @@ -30,7 +30,7 @@ public class Constant { */ public static String DEFAULT_DELETE_VALUE = "#[DELETE]#"; - //单次包最大2M + //单次包最大4M public static int MAX_LENGTH = 4 * 1024 * 1024; } diff --git a/dashboard/pom.xml b/dashboard/pom.xml index 40f6ef5..9675a2f 100644 --- a/dashboard/pom.xml +++ b/dashboard/pom.xml @@ -81,21 +81,6 @@ poi-ooxml 4.0.1 - - com.jd.common - sso-uim-spring - 1.2.0-SNAPSHOT - - - com.jd.dd - dd-open-gw-api - 4.1.0-SNAPSHOT - - - com.github.rholder - guava-retrying - 2.0.0 - diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java index 7ccab54..767387f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/controller/KeyController.java @@ -15,7 +15,7 @@ import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.KeyTimely; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.biz.service.KeyService; -import com.jd.platform.hotkey.dashboard.util.DateUtils; +import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.ExcelUtil; import org.springframework.stereotype.Controller; import org.springframework.ui.ModelMap; @@ -136,10 +136,10 @@ public class KeyController extends BaseController { public void export(HttpServletResponse resp,String startTime,String endTime,String app,String key){ SearchReq req = new SearchReq(); if(StringUtil.isNotEmpty(startTime)){ - req.setStartTime(DateUtils.strToDate(startTime)); + req.setStartTime(DateUtil.strToDate(startTime)); } if(StringUtil.isNotEmpty(endTime)){ - req.setEndTime(DateUtils.strToDate(endTime)); + req.setEndTime(DateUtil.strToDate(endTime)); } req.setApp(app); req.setKey(key); @@ -155,10 +155,10 @@ public class KeyController extends BaseController { public void exportList(HttpServletResponse resp,String startTime,String endTime,String app,String key){ SearchReq req = new SearchReq(); if(StringUtil.isNotEmpty(startTime)){ - req.setStartTime(DateUtils.strToDate(startTime)); + req.setStartTime(DateUtil.strToDate(startTime)); } if(StringUtil.isNotEmpty(endTime)){ - req.setEndTime(DateUtils.strToDate(endTime)); + req.setEndTime(DateUtil.strToDate(endTime)); } req.setApp(app); req.setKey(key); @@ -191,7 +191,7 @@ public class KeyController extends BaseController { List list = new ArrayList<>(); list.add(record.getKey()); list.add(record.getAppName()); - list.add(DateUtils.dateToStr(record.getCreateTime())); + list.add(DateUtil.dateToStr(record.getCreateTime())); rows.add(list); } return rows; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java index fddcaa4..d8a0555 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/biz/service/impl/KeyServiceImpl.java @@ -26,7 +26,7 @@ import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.biz.service.KeyService; import com.jd.platform.hotkey.dashboard.biz.service.RuleService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; -import com.jd.platform.hotkey.dashboard.util.DateUtils; +import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.PageUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import org.slf4j.Logger; @@ -79,12 +79,12 @@ public class KeyServiceImpl implements KeyService { } req.setApp(null); LocalDateTime now = LocalDateTime.now(); - req.setEndTime(req.getEndTime() == null ? DateUtils.ldtToDate(now) : req.getEndTime()); + req.setEndTime(req.getEndTime() == null ? DateUtil.ldtToDate(now) : req.getEndTime()); List rules = ruleService.listRules(null); if (type == 4) { - LocalDateTime st = req.getStartTime() == null ? now.minusMinutes(31) : DateUtils.dateToLdt(req.getStartTime()); - req.setStartTime(DateUtils.ldtToDate(st)); - LocalDateTime et = DateUtils.dateToLdt(req.getEndTime()); + LocalDateTime st = req.getStartTime() == null ? now.minusMinutes(31) : DateUtil.dateToLdt(req.getStartTime()); + req.setStartTime(DateUtil.ldtToDate(st)); + LocalDateTime et = DateUtil.dateToLdt(req.getEndTime()); boolean longTime = Duration.between(st, et).toHours() > 2; req.setType(longTime ? 6 : 5); List list = statisticsMapper.listOrderByTime(req); @@ -93,17 +93,17 @@ public class KeyServiceImpl implements KeyService { if (type == 5) { LocalDateTime startTime = now.minusMinutes(31); - req.setStartTime(DateUtils.ldtToDate(startTime)); + req.setStartTime(DateUtil.ldtToDate(startTime)); List list = statisticsMapper.listOrderByTime(req); return CommonUtil.processData(startTime, now, list, true, rules, app); } else if (type == 6) { LocalDateTime startTime2 = now.minusHours(25); - req.setStartTime(DateUtils.ldtToDate(startTime2)); + req.setStartTime(DateUtil.ldtToDate(startTime2)); List list2 = statisticsMapper.listOrderByTime(req); return CommonUtil.processData(startTime2, now, list2, false, rules, app); } else { LocalDateTime startTime3 = now.minusDays(7).minusHours(1); - req.setStartTime(DateUtils.ldtToDate(startTime3)); + req.setStartTime(DateUtil.ldtToDate(startTime3)); req.setType(6); List list3 = statisticsMapper.listOrderByTime(req); return CommonUtil.processData(startTime3, now, list3, false, rules, app); @@ -138,7 +138,7 @@ public class KeyServiceImpl implements KeyService { int hours = 6; // 默认查询6小时内的数据 SearchReq req = new SearchReq(); - req.setStartTime(DateUtils.preTime(hours)); + req.setStartTime(DateUtil.preTime(hours)); req.setEndTime(new Date()); List statistics = statisticsMapper.listStatistics(req); // 获取data Y轴 @@ -220,7 +220,7 @@ public class KeyServiceImpl implements KeyService { Map map = new HashMap<>(10); Map> listMap = statistics.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); for (Map.Entry> m : listMap.entrySet()) { - int start = DateUtils.preHoursInt(5); + int start = DateUtil.preHoursInt(5); map.put(m.getKey(), new int[hours]); int[] data = map.get(m.getKey()); int tmp = 0; @@ -249,7 +249,7 @@ public class KeyServiceImpl implements KeyService { private void checkParam(SearchReq req) { if (req.getStartTime() == null || req.getEndTime() == null) { - req.setStartTime(DateUtils.preTime(5)); + req.setStartTime(DateUtil.preTime(5)); req.setEndTime(new Date()); } /* long day = (req.getEndTime().getTime() - req.getStartTime().getTime()) / 86400000; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java index 1df046b..5154e43 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java @@ -1,6 +1,6 @@ package com.jd.platform.hotkey.dashboard.common.domain.req; -import com.jd.platform.hotkey.dashboard.util.DateUtils; +import com.jd.platform.hotkey.dashboard.util.DateUtil; import java.io.Serializable; import java.time.LocalDateTime; @@ -28,8 +28,8 @@ public class ChartReq implements Serializable { } public ChartReq(LocalDateTime st, LocalDateTime et, Integer limit) { - this.startTime = DateUtils.ldtToDate(st); - this.endTime = DateUtils.ldtToDate(et); + this.startTime = DateUtil.ldtToDate(st); + this.endTime = DateUtil.ldtToDate(et); this.limit = limit; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java index 8ee7f49..a63d987 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java @@ -1,7 +1,7 @@ package com.jd.platform.hotkey.dashboard.common.domain.req; import com.fasterxml.jackson.annotation.JsonFormat; -import com.jd.platform.hotkey.dashboard.util.DateUtils; +import com.jd.platform.hotkey.dashboard.util.DateUtil; import java.io.Serializable; import java.time.LocalDateTime; @@ -82,7 +82,7 @@ public class SearchReq implements Serializable { } public SearchReq(LocalDateTime st) { - this.startTime = DateUtils.ldtToDate(st); + this.startTime = DateUtil.ldtToDate(st); this.endTime = new Date(); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 537be41..4bd183b 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -2,8 +2,6 @@ package com.jd.platform.hotkey.dashboard.common.monitor; import cn.hutool.core.collection.CollectionUtil; -import cn.hutool.core.date.DateTime; -import cn.hutool.core.date.DateUtil; import com.alibaba.fastjson.JSON; import com.google.common.collect.Queues; import com.ibm.etcd.api.KeyValue; @@ -20,10 +18,11 @@ import com.jd.platform.hotkey.dashboard.common.domain.vo.AppCfgVo; import com.jd.platform.hotkey.dashboard.model.KeyRecord; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; -import com.jd.platform.hotkey.dashboard.util.DateUtils; +import com.jd.platform.hotkey.dashboard.util.DateUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; @@ -49,25 +48,20 @@ public class DataHandler { @Resource private IConfigCenter configCenter; - @Resource - private PushHandler pushHandler; - - private static final Integer CACHE_SIZE = 10000; /** * 队列 */ - private BlockingQueue RECORD_QUEUE = new LinkedBlockingQueue<>(); - + private BlockingQueue queue = new LinkedBlockingQueue<>(); /** * 入队 */ public void offer(IRecord record) { try { - RECORD_QUEUE.put(record); + queue.put(record); } catch (InterruptedException e) { e.printStackTrace(); } @@ -77,22 +71,25 @@ public class DataHandler { while (true) { try { List records = new ArrayList<>(); - Queues.drain(RECORD_QUEUE, records, CACHE_SIZE, 1, TimeUnit.SECONDS); + Queues.drain(queue, records, 1000, 1, TimeUnit.SECONDS); if (CollectionUtil.isEmpty(records)) { continue; } - List keyRecordList = new ArrayList<>(); + List keyRecordList = new ArrayList<>(records.size()); for (IRecord iRecord : records) { KeyRecord keyRecord = handHotKey(iRecord); if (keyRecord != null) { keyRecordList.add(keyRecord); } } + if(CollectionUtil.isEmpty(keyRecordList)){ + continue; + } keyRecordMapper.batchInsert(keyRecordList); } catch (Exception e) { log.error("batch insert error:{}", e.getMessage(), e); - e.printStackTrace(); +// e.printStackTrace(); } } @@ -123,7 +120,7 @@ public class DataHandler { */ private void putRecord(String app, String key, long createTime) { try { - RECORD_QUEUE.put(new IRecord() { + queue.put(new IRecord() { @Override public String appNameKey() { return app + "/" + key; @@ -187,85 +184,79 @@ public class DataHandler { /** * 每小时 统计一次record 表 结果记录到统计表 */ - @PostConstruct + @Scheduled(cron = "0 0 * * * ?") public void offlineStatistics() { - ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 - scheduledExecutorService.scheduleAtFixedRate(() -> { - try { - LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtils.ldtToDate(now); - int day = DateUtils.nowDay(now); - int hour = DateUtils.nowHour(now); - SearchReq preHour = new SearchReq(now.minusHours(1)); - List records = keyRecordMapper.maxHotKey(preHour); - if (records.size() != 0) { - records.forEach(x -> { - x.setBizType(1); - x.setCreateTime(nowTime); - x.setDays(day); - x.setHours(hour); - x.setUuid(1 + "_" + x.getKeyName() + "_" + hour); - }); - } - log.info("每小时统计最热点,时间：{}, 行数：{}", now.toString(), records.size()); - List statistics = keyRecordMapper.statisticsByRule(preHour); - if (statistics.size() != 0) { - statistics.forEach(x -> { - x.setBizType(6); - x.setRule(x.getRule()); - x.setCreateTime(nowTime); - x.setDays(day); - x.setHours(hour); - x.setUuid(6 + "_" + x.getKeyName() + "_" + hour); - }); - log.info("每小时统计规则,时间：{}, data list：{}", now.toString(), JSON.toJSONString(statistics)); - records.addAll(statistics); - } - if (records.size() > 0) { - int row = statisticsMapper.batchInsert(records); - } - } catch (Exception e) { - e.printStackTrace(); - } - }, 2, 1, TimeUnit.HOURS); - } - - /** - * 每分钟统计一次record 表 结果记录到统计表 - */ - @PostConstruct - public void offlineStatisticsRule() { - ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 - scheduledExecutorService.scheduleAtFixedRate(() -> { - try { - LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtils.ldtToDate(now); - int day = DateUtils.nowDay(now); - int hour = DateUtils.nowHour(now); - int minus = DateUtils.nowMinus(now); - - List records = keyRecordMapper.statisticsByRule(new SearchReq(now.minusMinutes(1))); - if (records.size() == 0) { - return; - } + try { + LocalDateTime now = LocalDateTime.now(); + Date nowTime = DateUtil.ldtToDate(now); + int day = DateUtil.nowDay(now); + int hour = DateUtil.nowHour(now); + SearchReq preHour = new SearchReq(now.minusHours(1)); + List records = keyRecordMapper.maxHotKey(preHour); + if (records.size() != 0) { records.forEach(x -> { - x.setBizType(5); + x.setBizType(1); + x.setCreateTime(nowTime); + x.setDays(day); + x.setHours(hour); + x.setUuid(1 + "_" + x.getKeyName() + "_" + hour); + }); + } + log.info("每小时统计最热点,时间：{}, 行数：{}", now.toString(), records.size()); + List statistics = keyRecordMapper.statisticsByRule(preHour); + if (statistics.size() != 0) { + statistics.forEach(x -> { + x.setBizType(6); x.setRule(x.getRule()); x.setCreateTime(nowTime); x.setDays(day); x.setHours(hour); - x.setMinutes(minus); - // 骚操作 临时解决没有rule字段的问题 - x.setUuid(5 + "_" + x.getKeyName() + "_" + minus); + x.setUuid(6 + "_" + x.getKeyName() + "_" + hour); }); + log.info("每小时统计规则,时间：{}, data list：{}", now.toString(), JSON.toJSONString(statistics)); + records.addAll(statistics); + } + if (records.size() > 0) { int row = statisticsMapper.batchInsert(records); -// log.info("每分钟统计规则，时间：{}, 影响行数：{}, data list:{}", now.toString(), row, JSON.toJSONString(records)); - } catch (Exception e) { - e.printStackTrace(); } - }, 2, 1, TimeUnit.MINUTES); + } catch (Exception e) { + e.printStackTrace(); + } + + } + + + /** + * 每分钟统计一次record 表 结果记录到统计表 + */ + @Scheduled(cron = "0 */1 * * * ?") + public void offlineStatisticsRule() { + try { + LocalDateTime now = LocalDateTime.now(); + Date nowTime = DateUtil.ldtToDate(now); + int day = DateUtil.nowDay(now); + int hour = DateUtil.nowHour(now); + int minus = DateUtil.nowMinus(now); + + List records = keyRecordMapper.statisticsByRule(new SearchReq(now.minusMinutes(1))); + if (records.size() == 0) { + return; + } + records.forEach(x -> { + x.setBizType(5); + x.setRule(x.getRule()); + x.setCreateTime(nowTime); + x.setDays(day); + x.setHours(hour); + x.setMinutes(minus); + // 骚操作 临时解决没有rule字段的问题 + x.setUuid(5 + "_" + x.getKeyName() + "_" + minus); + }); + int row = statisticsMapper.batchInsert(records); +// log.info("每分钟统计规则，时间：{}, 影响行数：{}, data list:{}", now.toString(), row, JSON.toJSONString(records)); + } catch (Exception e) { + e.printStackTrace(); + } } @@ -273,36 +264,30 @@ public class DataHandler { /** * 每天根据app的配置清理过期数据 */ - @PostConstruct + @Scheduled(cron = "0 0 1 * * ?") public void clearExpireData() { - ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 - scheduledExecutorService.scheduleAtFixedRate(() -> { - List keyValues = configCenter.getPrefix(ConfigConstant.appCfgPath); + try { + LocalDateTime now = LocalDateTime.now(); + List keyValues = configCenter.getPrefix(ConfigConstant.clearCfgPath); for (KeyValue kv : keyValues) { - try { - String val = kv.getValue().toStringUtf8(); - AppCfgVo cfg = JSON.parseObject(val, AppCfgVo.class); - String app = cfg.getApp(); - //保存几天 - Integer days = cfg.getDataTtl(); - DateTime dateTime = DateUtil.offsetDay(new Date(), -days); - - - summaryMapper.clearExpireData(app, dateTime); - keyRecordMapper.clearExpireData(app, dateTime); - statisticsMapper.clearExpireData(app, dateTime); - } catch (Exception e) { - e.printStackTrace(); - } + String key = kv.getKey().toStringUtf8(); + String ttl = kv.getValue().toStringUtf8(); + String app = key.replace(ConfigConstant.clearCfgPath, ""); + Date expireDate = DateUtil.ldtToDate(now.minusDays(Integer.parseInt(ttl))); + summaryMapper.clearExpireData(app, expireDate); + keyRecordMapper.clearExpireData(app, expireDate); + statisticsMapper.clearExpireData(app, expireDate); } - }, 1, 5, TimeUnit.MINUTES); + } catch (Exception e) { + e.printStackTrace(); + } } /** * 每10秒检测一次热点记录 用于监控报警 */ + //TODO WANG @PostConstruct public void scanRecordForMonitor() { ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); @@ -353,7 +338,9 @@ public class DataHandler { int threshold = type == 1 ? cfg.getWarnMax() : cfg.getWarnMin(); String time = LocalDateTime.now().toString().replace("T", " "); String content = String.format("【警报】 应用：【%s】 热点记录在%d秒内累计: %d, %s阈值: %d \n【时间】：%s", cfg.getApp(), cfg.getWarnPeriod(), count, str, threshold, time); - pushHandler.pushMsg(cfg.getApp(), date, content); + //TODO WANG + System.out.println(content); +// pushHandler.pushMsg(cfg.getApp(), date, content); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java deleted file mode 100644 index fdb022d..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/PushHandler.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.jd.platform.hotkey.dashboard.common.monitor; - - -import com.jd.platform.hotkey.dashboard.biz.mapper.UserMapper; -import com.jd.platform.hotkey.dashboard.warn.DongDongApiManager; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; - -import javax.annotation.Resource; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -/** - * @ProjectName: hotkey - * @ClassName: PushHandler - * @Description: 处理推送 - * @Author: liyunfeng31 - * @Date: 2020/9/3 10:47 - */ - -@Component -public class PushHandler { - - private Logger logger = LoggerFactory.getLogger(getClass()); - - - /** - * 报警标题 - */ - private static final String TITLE = "hotKey异常提醒"; - - /** - * 拦截重复报警间隔 2分钟 - */ - private static final long INTERVAL = 2 * 60 * 1000L; - - - /** - * app-time 用于app存储报警时间 做拦截 - */ - private Map appIntervalMap = new ConcurrentHashMap<>(); - - - @Resource - private DongDongApiManager apiManager; - - @Resource - private UserMapper userMapper; - - - /** - * 发送消息 - */ - public void pushMsg(String app, Date msgTime, String content) { - boolean send = check(app, msgTime.getTime()); - if (send) { - logger.info("Warn PushMsg content:{}", content); - List erpList = userMapper.listErpByApp(app); - apiManager.push(TITLE, content, erpList); - } - } - - - /** - * 防止重复发送警报 - * - * @param warnApp app - * @param msgTime time - * @return result - */ - private synchronized boolean check(String warnApp, Long msgTime) { - Long maxTime = appIntervalMap.get(warnApp); - if (maxTime == null) { - appIntervalMap.put(warnApp, msgTime + INTERVAL); - return true; - } else { - if (msgTime >= maxTime) { - appIntervalMap.put(warnApp, msgTime + INTERVAL); - return true; - } - } - return false; - } - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpConfiguration.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpConfiguration.java deleted file mode 100644 index 6b12a85..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpConfiguration.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.jd.platform.hotkey.dashboard.erp; - - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class ErpConfiguration { - - @Autowired - ErpProperties erpProperties; - - @Bean - public ErpUimInterceptor erpUimInterceptor() { - ErpUimInterceptor springSSOInterceptor = new ErpUimInterceptor(); - springSSOInterceptor.setSsoAppKey(erpProperties.getSsoAppKey()); - springSSOInterceptor.setSsoAppToken(erpProperties.getSsoAppToken()); - springSSOInterceptor.setLoginUrl(erpProperties.getLoginUrl()); - springSSOInterceptor.setExcludePath(erpProperties.getExcludePath()); - springSSOInterceptor.setSsoAppUrl(erpProperties.getSsoAppUrl()); - return springSSOInterceptor; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpProperties.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpProperties.java deleted file mode 100644 index 1d4cf23..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpProperties.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.jd.platform.hotkey.dashboard.erp; - -import com.jd.platform.hotkey.dashboard.autoconfigure.AbstractProperties; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.stereotype.Component; - -/** - * User: fuxueliang - * Date: 16/8/17 - * Email: fuxueliang@jd.com - */ -@Component -@ConfigurationProperties(prefix = "erp") -public class ErpProperties extends AbstractProperties { - - private String excludePath; - - private String loginUrl; - - private String ssoAppUrl; - - private String ssoAppKey; - - private String ssoAppToken; - - private Boolean enabled; - - public String getExcludePath() { - return excludePath; - } - - public void setExcludePath(String excludePath) { - this.excludePath = excludePath; - } - - public String getLoginUrl() { - return loginUrl; - } - - public void setLoginUrl(String loginUrl) { - this.loginUrl = loginUrl; - } - - public String getSsoAppUrl() { - return ssoAppUrl; - } - - public void setSsoAppUrl(String ssoAppUrl) { - this.ssoAppUrl = ssoAppUrl; - } - - public String getSsoAppKey() { - return ssoAppKey; - } - - public void setSsoAppKey(String ssoAppKey) { - this.ssoAppKey = ssoAppKey; - } - - public String getSsoAppToken() { - return ssoAppToken; - } - - public void setSsoAppToken(String ssoAppToken) { - this.ssoAppToken = ssoAppToken; - } - - public Boolean getEnabled() { - return enabled; - } - - public void setEnabled(Boolean enabled) { - this.enabled = enabled; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java deleted file mode 100644 index 2bd7653..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.jd.platform.hotkey.dashboard.erp; - -import com.jd.common.springmvc.interceptor.SpringSSOInterceptor; -import com.jd.common.web.LoginContext; -import com.jd.platform.hotkey.dashboard.model.User; -import com.jd.platform.hotkey.dashboard.biz.service.UserService; -import org.springframework.beans.factory.annotation.Autowired; - -import javax.servlet.http.Cookie; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - - -public class ErpUimInterceptor extends SpringSSOInterceptor { - - @Autowired - private UserService userService; - - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) throws Exception { - LoginContext loginContext = LoginContext.getLoginContext(); - if (null == loginContext) { - super.preHandle(request, response, handler); - } - loginContext = LoginContext.getLoginContext(); - if (null == loginContext) { - return true; - } - /*Cookie cookie = new Cookie("erp", loginContext.getNick()); - cookie.setMaxAge(3600*24*7); - cookie.setPath("/"); - response.addCookie(cookie);*/ - String pin = loginContext.getPin(); - String nickName = loginContext.getNick(); - String orgName = loginContext.getOrgName(); - String orgId = loginContext.getOrgId(); - String mobile = loginContext.getMobile(); - String email = loginContext.getEmail(); - User user = new User(); - user.setAppName("test"); - user.setUserName(pin); - user.setNickName(nickName); - user.setPhone(mobile); - user.setRole("APPUSER"); - Cookie cookie = userService.loginErpUser(user); - Cookie[] cookies = request.getCookies(); - if(cookies != null){ - for(Cookie tempCookie : cookies){ - if("token".equals(tempCookie.getName()) && tempCookie.getValue() != null){ - return true; - } - } - } - response.addCookie(cookie); - return true; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java index 2f3bc11..8876162 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/etcd/EtcdMonitor.java @@ -6,13 +6,13 @@ import com.ibm.etcd.api.KeyValue; import com.ibm.etcd.client.kv.KvClient; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.IRecord; import com.jd.platform.hotkey.dashboard.common.monitor.DataHandler; import com.jd.platform.hotkey.dashboard.biz.mapper.SummaryMapper; -import com.jd.platform.hotkey.dashboard.common.monitor.PushHandler; import com.jd.platform.hotkey.dashboard.model.Worker; import com.jd.platform.hotkey.dashboard.netty.HotKeyReceiver; import com.jd.platform.hotkey.dashboard.biz.service.WorkerService; @@ -25,6 +25,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; import javax.annotation.Resource; import java.util.*; import java.util.concurrent.ExecutorService; @@ -54,11 +55,8 @@ public class EtcdMonitor { @Resource private DataHandler dataHandler; - @Resource - private PushHandler pushHandler; - - public static final ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(16); + private final ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(16); /** * 监听新来的热key，该key的产生是来自于手工在控制台添加 @@ -146,7 +144,41 @@ public class EtcdMonitor { */ private void dealHotKey() { threadPoolExecutor.submit(() -> { - dataHandler.dealHotKey(); + while (true) { + try { + //获取发来的这个热key，存入本地caffeine，设置过期时间 + HotKeyModel model = HotKeyReceiver.take(); + + //将该key放入实时热key本地缓存中 + HotKeyReceiver.writeToLocalCaffeine(model); + + dataHandler.offer(new IRecord() { + @Override + public String appNameKey() { + return model.getAppName() + "/" + model.getKey(); + } + + @Override + public String value() { + return UUID.randomUUID().toString(); + } + + @Override + public int type() { + return 0; + } + + @Override + public Date createTime() { + return new Date(); + } + }); + + } catch (Exception e) { + e.printStackTrace(); + } + } + }); } @@ -180,8 +212,6 @@ public class EtcdMonitor { private void fetchRuleFromEtcd() { RuleUtil.init(); try { - List ruleList = new ArrayList<>(); - //从etcd获取rule List keyValues = configCenter.getPrefix(ConfigConstant.rulePath); @@ -265,4 +295,13 @@ public class EtcdMonitor { return watchIterator.next().getEvents().get(0); } + @PreDestroy + public void close() { + try { + threadPoolExecutor.shutdown(); + } catch (Exception e) { + log.error("EtcdMonitor threadPoolExecutor shutdown exception:{}", e.getMessage(), e); + } + } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java index 8fcd977..1f74e2a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java @@ -10,6 +10,7 @@ import org.springframework.http.HttpMethod; import org.springframework.util.StringUtils; import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; +import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; @@ -27,9 +28,8 @@ public class JwtInterceptor extends HandlerInterceptorAdapter{ } String header = request.getHeader("x-requested-with"); if(!StringUtils.isEmpty(header) && "XMLHttpRequest".endsWith(header) && request.getMethod().equals(Constant.POST)){ - final String authHeader = JwtTokenUtil.getAuthHeader(request); - if (StringUtils.isEmpty(authHeader) - || !authHeader.startsWith(JwtTokenUtil.TOKEN_PREFIX)) { + String authHeader = request.getHeader(JwtTokenUtil.AUTH_HEADER_KEY); + if (StringUtils.isEmpty(authHeader) || !authHeader.startsWith(JwtTokenUtil.TOKEN_PREFIX)) { throw new BizException(ResultEnum.NO_LOGIN); } final String token = authHeader.substring(2); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java index 529e416..7360a63 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java @@ -1,22 +1,18 @@ package com.jd.platform.hotkey.dashboard.interceptor; -import com.jd.platform.hotkey.dashboard.erp.ErpUimInterceptor; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.core.Ordered; import org.springframework.web.servlet.config.annotation.*; + @Configuration public class MyWebAppConfigurer implements WebMvcConfigurer { - @Autowired - ErpUimInterceptor erpUimInterceptor; @Override public void addInterceptors(InterceptorRegistry registry) { //拦截路径可自行配置多个 可用 ，分隔开 InterceptorRegistration registration = registry.addInterceptor(new JwtInterceptor()).addPathPatterns("/**"); - registry.addInterceptor(erpUimInterceptor); - registration.excludePathPatterns("/user/login","/user/getUserName","/error","/static/**","/main/**"); + registration.excludePathPatterns("/user/login","/error","/static/**","/main/**"); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index deda3f8..350c3b1 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.dashboard.util; +import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.model.Summary; @@ -53,6 +54,10 @@ public class CommonUtil { } + public static String keyName(String k) { + int index = k.lastIndexOf("/"); + return k.substring(index + 1); + } public static String encoder(String text) { @@ -76,6 +81,49 @@ public class CommonUtil { } + /** + * 拼装数据 + * + * @param list list-data + * @param startTime 开始时间 + * @param size 格子数 + * @param type 类型 1分钟 2小时 + * @return vo + */ + public static HotKeyLineChartVo assembleData(List list, LocalDateTime startTime, int size, int type) { + Set set = new TreeSet<>(); + boolean isHour = type == 1; + String suffix = isHour ? "60" : "24"; + String pattern = isHour ? DateUtil.PATTERN_MINUS : DateUtil.PATTERN_HOUR; + Map map = new HashMap<>(10); + Map> listMap = listGroup(list); +// log.info("按照rule分组以后的listMap--> {}", JSON.toJSONString(listMap)); + for (Map.Entry> m : listMap.entrySet()) { + int start = DateUtil.reviseTime(startTime, 0, type); + map.put(m.getKey(), new int[size]); + int[] data = map.get(m.getKey()); + int tmp = 0; + for (int i = 0; i < size; i++) { + if (String.valueOf(start).endsWith(suffix)) { + LocalDateTime tmpTime = DateUtil.strToLdt((start - 1) + "", pattern); + start = DateUtil.reviseTime(tmpTime, 1, type); + } +// log.info("start--> {}, tmp---> {} ", start, tmp); + set.add(DateUtil.strToLdt(start + "", pattern).toString().replace("T", " ")); + Statistics st = m.getValue().get(tmp); + int val = isHour ? st.getMinutes() : st.getHours(); + if (start != val) { + data[i] = 0; + } else { + tmp++; + data[i] = st.getCount(); + } + start++; + } + } + return new HotKeyLineChartVo(new ArrayList<>(set), map); + } + /** * 分组 @@ -86,6 +134,14 @@ public class CommonUtil { private static Map> listGroup(List list) { // return list.stream().collect(Collectors.groupingBy(Statistics::getRule)); return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); + + /* + // master code + if (Constant.VERSION != 1) { + return list.stream().collect(Collectors.groupingBy(Statistics::getRule)); + } + return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); + */ } /** @@ -117,10 +173,10 @@ public class CommonUtil { Duration duration = Duration.between(st, et); long passTime = isMinute ? duration.toMinutes() : duration.toHours(); Map timeCountMap = new TreeMap<>(); - String pattern = isMinute ? DateUtils.PATTERN_MINUS : DateUtils.PATTERN_HOUR; + String pattern = isMinute ? DateUtil.PATTERN_MINUS : DateUtil.PATTERN_HOUR; for (int i = 1; i < passTime; i++) { - int time = DateUtils.reviseTime(st, i, isMinute ? 1 : 2); - xAxisSet.add(DateUtils.formatTime(time, pattern)); + int time = DateUtil.reviseTime(st, i, isMinute ? 1 : 2); + xAxisSet.add(DateUtil.formatTime(time, pattern)); timeCountMap.put(time, null); } Map> ruleStatsMap = listGroup(list); @@ -174,12 +230,12 @@ public class CommonUtil { String app = args[0]; String rule = args[1]; String hitTime = args[2]; - Date time = DateUtils.strToDate(hitTime); + Date time = DateUtil.strToDate(hitTime); assert time != null; - LocalDateTime ldt = DateUtils.dateToLdt(time); - int day = DateUtils.nowDay(ldt); - int hour = DateUtils.nowHour(ldt); - int minus = DateUtils.nowMinus(ldt); + LocalDateTime ldt = DateUtil.dateToLdt(time); + int day = DateUtil.nowDay(ldt); + int hour = DateUtil.nowHour(ldt); + int minus = DateUtil.nowMinus(ldt); long seconds = time.getTime() / 1000; String[] counts = map.get(key).split("-"); int hitCount = Integer.parseInt(counts[0]); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java similarity index 99% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java index c5c8c53..ce7a1ff 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtils.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java @@ -10,7 +10,7 @@ import java.util.Date; /** * @author liyunfeng31 */ -public class DateUtils { +public class DateUtil { public static final String PATTERN_SECONDS="yyMMddHHmmss"; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java deleted file mode 100644 index 36f6b7c..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongApiManager.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; -import org.thymeleaf.util.StringUtils; - -import javax.annotation.PostConstruct; -import javax.annotation.Resource; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -/** - * @author lihongliang32 - * @date 2020/5/20 5:53 下午 - */ -@Service -public class DongDongApiManager { - - @Resource - private DongDongUtil dongUtil; - - - private Logger log = LoggerFactory.getLogger(getClass()); - - /** - * 启动时获取token，每隔1小时刷新一次token - */ - @PostConstruct - public void updateAccessToken() { - ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); - //开启拉取etcd的worker信息，如果拉取失败，则定时继续拉取 - scheduledExecutorService.scheduleAtFixedRate(() -> { - try { - String accessToken = dongUtil.grant(); - log.info("刷新token：" + accessToken); - DongDongToken.TOKEN = accessToken; - } catch (Exception e) { - log.error("refreshAccessSignature error:", e); - } - }, 0, 60, TimeUnit.MINUTES); - - } - - - public boolean push(String title, String content, List erpList) { - String accessToken = DongDongToken.TOKEN; - if (StringUtils.isEmpty(accessToken)) { - return false; - } - try { - return dongUtil.push(title, content, accessToken, erpList); - } catch (Exception e) { - return false; - } - } - -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongToken.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongToken.java deleted file mode 100644 index b3cd438..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongToken.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-11-02 - */ -public class DongDongToken { - public static String TOKEN = ""; -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java deleted file mode 100644 index 7ec7d85..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/warn/DongDongUtil.java +++ /dev/null @@ -1,119 +0,0 @@ -package com.jd.platform.hotkey.dashboard.warn; - -import cn.hutool.json.JSONObject; -import com.alibaba.fastjson.JSON; -import com.jd.dd.open.gw.api.domain.AccessSignatureResult; -import com.jd.dd.open.gw.api.domain.MessagePushResult; -import org.apache.http.NameValuePair; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.EntityUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -/** - * @author liyunfeng31 - */ -@Component -public class DongDongUtil { - - - private Logger log = LoggerFactory.getLogger(getClass()); - - @Value("${dongdong.aspId:110.200.0000002.266}") - private String aspId; - - @Value("${dongdong.secret:e2b05aa43959ddb6d37bd056b0243bc9}") - private String secret; - - @Value("${dongdong.noticeId:~hotkey}") - private String noticeId; - - - private final String VERSION="4.3"; - - - public String grant() throws Exception{ - String dogUrl = "http://dog-ee.jd.com/grant"; - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("aspid", aspId)); - params.add(new BasicNameValuePair("secret", secret)); - params.add(new BasicNameValuePair("version", VERSION)); - - CloseableHttpClient httpclient = HttpClients.createDefault(); - HttpPost httpPost = new HttpPost(dogUrl); - //传参 - httpPost.setEntity(new UrlEncodedFormEntity(params)); - - CloseableHttpResponse response = httpclient.execute(httpPost); - String res = EntityUtils.toString(response.getEntity()); - log.info("TASK grantAccessSign param:{} result:{}",JSON.toJSONString(params),res); - //返回结果对象AccessSignatureResult - AccessSignatureResult result = JSON.parseObject(res, AccessSignatureResult.class); - if (result.getCode() == 230031) { - //请求成功 - return result.getAccessToken(); - } - else { - //失败操作 - return String.valueOf(result.getCode()); - } - - } - - public boolean push(String title,String content,String accessToken, List erpList) throws Exception{ - String dogUrl = "http://dog-ee.jd.com/push"; - List params = new ArrayList<>(); - params.add(new BasicNameValuePair("accessid", UUID.randomUUID().toString())); - params.add(new BasicNameValuePair("accessToken", accessToken)); - params.add(new BasicNameValuePair("aspid", aspId));//需要申请 - params.add(new BasicNameValuePair("version", VERSION)); - params.add(new BasicNameValuePair("timestamp", String.valueOf(System.currentTimeMillis()))); - - //消息相关参数 - JSONObject json = new JSONObject(); - json.put("type", "notice_message"); - json.put("ver", VERSION); - json.put("title", title); - json.put("content", content); - json.put("noticeId", noticeId); - json.put("toTerminal", 7); - json.put("sla", 0); - json.put("app", "ee"); - //接收方 - List pins = new ArrayList<>(); - pins.add("wuweifeng10"); - pins.add("liyunfeng31"); - pins.addAll(erpList); - json.put("tos", pins); - - JSONObject extend = new JSONObject(); - extend.put("url", "http://hotkey.jd.com"); - /* extend.writeToLocalCaffeine("pic", "https://img14.360buyimg.com/imagetools/jfs/t1/144156/7/8194/7263/5f5b34baEdfd90a49/a15144ab447bfbf2.png");*/ - json.put("extend", extend); - params.add(new BasicNameValuePair("jsonMsg",JSON.toJSONString(json))); - - CloseableHttpClient httpclient = HttpClients.createDefault(); - HttpPost httpPost = new HttpPost(dogUrl); - //传参 - httpPost.setEntity(new UrlEncodedFormEntity(params)); - httpPost.setEntity(new UrlEncodedFormEntity(params,"UTF-8")); - CloseableHttpResponse response = httpclient.execute(httpPost); - String res = EntityUtils.toString(response.getEntity()); - log.info("发送报警信息参数：{} 结果：{}", JSON.toJSONString(params), res); - //返回结果对象AccessSignatureResult - MessagePushResult result = JSON.parseObject(res, MessagePushResult.class); - return result.getCode() == 230070; - } - -} diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index 66bb0d7..987fbbf 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -29,7 +29,7 @@ spring : datasource: username: ${MYSQL_USER:root} password: ${MYSQL_PASS:root} - url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/luck?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT + url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/hotkey_db?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT driver-class-name: com.mysql.cj.jdbc.Driver hikari: max-lifetime: 120000 @@ -48,11 +48,4 @@ pagehelper: etcd: server: ${etcdServer:http://127.0.0.1:2379} -erp: - enabled: false - ssoAppKey: xxx - ssoAppToken: xxx - loginUrl: https://ssa.jd.com/sso/login - excludePath: /login,/assets,/bootstarp,/common,/htmlformat,/images,/jn,/jquery,/user/login,login - ssoAppUrl: http://ssa.jd.com - defaultPwd: xxx + diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml index eef4c75..bc93f24 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/biz/mapper/KeyRecordMapper.xml @@ -197,19 +197,23 @@ diff --git a/dashboard/src/main/resources/logback-spring.xml b/dashboard/src/main/resources/logback-spring.xml index f97e83e..14649ec 100644 --- a/dashboard/src/main/resources/logback-spring.xml +++ b/dashboard/src/main/resources/logback-spring.xml @@ -6,7 +6,7 @@ debug：当此属性设置为true时，将打印出logback内部日志信息， --> - + diff --git a/dashboard/src/main/resources/static/admin/assets/css/extra-pages.css b/dashboard/src/main/resources/static/admin/assets/css/extra-pages.css index c067452..54ca0fc 100644 --- a/dashboard/src/main/resources/static/admin/assets/css/extra-pages.css +++ b/dashboard/src/main/resources/static/admin/assets/css/extra-pages.css @@ -333,7 +333,7 @@ h2.headline.text-info { .lockscreen > body { background: transparent; } -/* We will writeToLocalCaffeine the dynamically generated digital clock here */ +/* We will put the dynamically generated digital clock here */ .lockscreen .headline { color: #fff; text-shadow: 1px 3px 5px rgba(0, 0, 0, 0.5); diff --git a/dashboard/src/main/resources/static/admin/common/js/key-add.js b/dashboard/src/main/resources/static/admin/common/js/key-add.js index f9aadfe..4cf1263 100644 --- a/dashboard/src/main/resources/static/admin/common/js/key-add.js +++ b/dashboard/src/main/resources/static/admin/common/js/key-add.js @@ -14,6 +14,10 @@ $.ajax({ async : false, error : function(XMLHttpRequest){ $.modal.alertError(XMLHttpRequest.responseJSON.msg); + let token = getCookie("token"); + if(XMLHttpRequest.status == 1000 && ( token == "undefined" || token =="")){ + top.location.href = '/user/login'; + } }, success : function(data) { console.log(data) diff --git a/dashboard/src/main/resources/static/admin/common/js/key-edit.js b/dashboard/src/main/resources/static/admin/common/js/key-edit.js index 3579fbf..edcb793 100644 --- a/dashboard/src/main/resources/static/admin/common/js/key-edit.js +++ b/dashboard/src/main/resources/static/admin/common/js/key-edit.js @@ -18,6 +18,10 @@ function edit() { async : false, error : function(XMLHttpRequest){ $.modal.alertError(XMLHttpRequest.responseJSON.msg); + let token = getCookie("token"); + if(XMLHttpRequest.status == 1000 && ( token == "undefined" || token =="")){ + top.location.href = '/user/login'; + } }, success : function(data) { $.operate.saveSuccess(data); diff --git a/dashboard/src/main/resources/static/admin/common/js/rule-add.js b/dashboard/src/main/resources/static/admin/common/js/rule-add.js index fc87f35..dc07324 100644 --- a/dashboard/src/main/resources/static/admin/common/js/rule-add.js +++ b/dashboard/src/main/resources/static/admin/common/js/rule-add.js @@ -17,6 +17,10 @@ function add() { async : false, error : function(XMLHttpRequest){ $.modal.alertError(XMLHttpRequest.responseJSON.msg); + let token = getCookie("token"); + if(XMLHttpRequest.status == 1000 && ( token == "undefined" || token =="")){ + top.location.href = '/user/login'; + } }, success : function(data) { $.operate.saveSuccess(data); diff --git a/dashboard/src/main/resources/static/admin/common/js/rule-edit.js b/dashboard/src/main/resources/static/admin/common/js/rule-edit.js index 46a1115..1ad1590 100644 --- a/dashboard/src/main/resources/static/admin/common/js/rule-edit.js +++ b/dashboard/src/main/resources/static/admin/common/js/rule-edit.js @@ -16,9 +16,11 @@ function edit() { "Authorization":getCookie("token") }, async : false, - error : function(response,status,xhr) { - - $.modal.alertError("系统错误"); + error : function(XMLHttpRequest) { + let token = getCookie("token"); + if(XMLHttpRequest.status == 1000 && ( token == "undefined" || token =="")){ + top.location.href = '/user/login'; + } }, success : function(data) { $.operate.saveSuccess(data); diff --git a/dashboard/src/main/resources/static/admin/common/js/user-add.js b/dashboard/src/main/resources/static/admin/common/js/user-add.js index 9f5103f..7c2b6fe 100644 --- a/dashboard/src/main/resources/static/admin/common/js/user-add.js +++ b/dashboard/src/main/resources/static/admin/common/js/user-add.js @@ -32,6 +32,10 @@ function add() { async : false, error : function(XMLHttpRequest){ $.modal.alertError(XMLHttpRequest.responseJSON.msg); + let token = getCookie("token"); + if(XMLHttpRequest.status == 1000 && ( token == "undefined" || token =="")){ + top.location.href = '/user/login'; + } }, success : function(data) { $.operate.saveSuccess(data); diff --git a/dashboard/src/main/resources/static/admin/common/js/user-edit.js b/dashboard/src/main/resources/static/admin/common/js/user-edit.js index b147d59..3b95013 100644 --- a/dashboard/src/main/resources/static/admin/common/js/user-edit.js +++ b/dashboard/src/main/resources/static/admin/common/js/user-edit.js @@ -18,6 +18,10 @@ function edit() { async : false, error : function(XMLHttpRequest){ $.modal.alertError(XMLHttpRequest.responseJSON.msg); + let token = getCookie("token"); + if(XMLHttpRequest.status == 1000 && ( token == "undefined" || token =="")){ + top.location.href = '/user/login'; + } }, success : function(data) { $.operate.saveSuccess(data); diff --git a/dashboard/src/main/resources/templates/admin/common/html/footer.html b/dashboard/src/main/resources/templates/admin/common/html/footer.html index 662d4f3..703a359 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/footer.html +++ b/dashboard/src/main/resources/templates/admin/common/html/footer.html @@ -10,8 +10,8 @@ - 2020 京东零售-平台业务中心-平台业务研发部-基础业务研发部 版权所有 | 使用支持：武伟峰 | 咚咚：85134158 -
接入文档：https://cf.jd.com/pages/viewpage.action?pageId=277756998 +
diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index 4764768..39d52f1 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -60,10 +60,10 @@ 变更记录 - - 应用配置 + 数据清理配置 diff --git a/dashboard/src/main/resources/templates/admin/common/html/title.html b/dashboard/src/main/resources/templates/admin/common/html/title.html index 08cf7c8..aa2fcaf 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/title.html +++ b/dashboard/src/main/resources/templates/admin/common/html/title.html @@ -36,8 +36,8 @@
Widget
+ -