From 403bce89eee08511a7da68805a0b431bf8b00a23 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Thu, 11 Jun 2020 18:40:00 +0800 Subject: [PATCH 01/51] =?UTF-8?q?=E9=A6=96=E9=A1=B5=E5=A4=A7=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/common/domain/req/ChartReq.java | 4 +- .../common/domain/req/SearchReq.java | 12 +- .../dashboard/common/monitor/DataHandler.java | 4 +- .../dashboard/common/monitor/EtcdMonitor.java | 2 +- .../dashboard/controller/RuleController.java | 9 + .../hotkey/dashboard/service/RuleService.java | 4 + .../service/impl/KeyServiceImpl.java | 190 ++++++++++-------- .../service/impl/RuleServiceImpl.java | 20 +- .../hotkey/dashboard/util/CommonUtil.java | 102 ++++++++-- .../hotkey/dashboard/util/DateUtil.java | 57 ++++-- .../admin/assets/js/bootstrap/js/base_list.js | 4 +- .../main/resources/templates/admin/main.html | 164 ++++++++++++--- .../resources/templates/admin/mainbak.html | 106 ++++++++++ 13 files changed, 530 insertions(+), 148 deletions(-) create mode 100644 dashboard/src/main/resources/templates/admin/mainbak.html diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java index 6b0b4f1..5154e43 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/ChartReq.java @@ -28,8 +28,8 @@ public class ChartReq implements Serializable { } public ChartReq(LocalDateTime st, LocalDateTime et, Integer limit) { - this.startTime = DateUtil.localDateTimeToDate(st); - this.endTime = DateUtil.localDateTimeToDate(et); + this.startTime = DateUtil.ldtToDate(st); + this.endTime = DateUtil.ldtToDate(et); this.limit = limit; } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java index 4e7a462..57b7f0a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java @@ -6,6 +6,7 @@ import com.jd.platform.hotkey.dashboard.util.DateUtil; import java.io.Serializable; import java.time.LocalDateTime; import java.util.Date; +import java.util.List; /** * @ProjectName: hotkey @@ -30,6 +31,8 @@ public class SearchReq implements Serializable { private String key; + private String rules; + public SearchReq() { } @@ -81,9 +84,16 @@ public class SearchReq implements Serializable { this.key = key; } + public String getRules() { + return rules; + } + + public void setRules(String rules) { + this.rules = rules; + } public SearchReq(LocalDateTime st) { - this.startTime = DateUtil.localDateTimeToDate(st); + this.startTime = DateUtil.ldtToDate(st); this.endTime = new Date(); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 5ac7073..4308467 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -136,7 +136,7 @@ public class DataHandler { try { // 每小时 统计一次record 表 结果记录到统计表 LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtil.localDateTimeToDate(now); + Date nowTime = DateUtil.ldtToDate(now); int day = DateUtil.nowDay(now); int hour = DateUtil.nowHour(now); @@ -165,7 +165,7 @@ public class DataHandler { try { // 每分钟小时 统计一次record 表 结果记录到统计表 LocalDateTime now = LocalDateTime.now(); - Date nowTime = DateUtil.localDateTimeToDate(now); + Date nowTime = DateUtil.ldtToDate(now); int day = DateUtil.nowDay(now); int hour = DateUtil.nowHour(now); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java index 26e5d39..680d6d5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java @@ -36,7 +36,7 @@ import java.util.concurrent.CompletableFuture; * @Date: 2020/4/18 18:29 */ @SuppressWarnings("ALL") -@Component +//@Component public class EtcdMonitor { private static Logger log = LoggerFactory.getLogger(EtcdMonitor.class); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java index ba80e85..bd5a66a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java @@ -14,6 +14,7 @@ import org.springframework.ui.ModelMap; import org.springframework.web.bind.annotation.*; import javax.annotation.Resource; +import java.util.List; @Controller @@ -107,5 +108,13 @@ public class RuleController extends BaseController { return "admin/rule/view"; } + + @PostMapping("/listRules") + @ResponseBody + public List rules(){ + List info = ruleService.listRules(null); + return info; + } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java index 8c6ca10..ebd974e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java @@ -6,6 +6,8 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.KeyRule; import com.jd.platform.hotkey.dashboard.model.Rules; +import java.util.List; + /** * @ProjectName: hotkey * @ClassName: RuleService @@ -27,4 +29,6 @@ public interface RuleService { PageInfo pageKeyRule(PageReq page, SearchReq param); int save(Rules rules); + + List listRules(String app); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 9201180..2bf3a06 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.dashboard.service.impl; +import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.ibm.etcd.api.Event; @@ -11,8 +12,6 @@ import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; -import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; -import com.jd.platform.hotkey.dashboard.common.ex.BizException; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.ReceiveCountMapper; @@ -54,82 +53,45 @@ public class KeyServiceImpl implements KeyService { @Resource private StatisticsMapper statisticsMapper; - - public HotKeyLineChartVo ruleLineChart2(SearchReq req) { - int type = req.getType(); - if(req.getEndTime() == null){ - req.setEndTime(new Date()); - } - switch (type){ - case 1: - req.setStartTime(DateUtil.preMinus(30)); - List list = statisticsList(); - System.out.println("30 min"); - break; - case 2: - req.setStartTime(DateUtil.preDays(1)); - System.out.println("24 hours"); - break; - case 3: - req.setStartTime(DateUtil.preDays(7)); - System.out.println("7 days"); - break; - default: - System.out.println("============="); - } - return null; + public static void main(String[] args) { + HotKeyLineChartVo result = new KeyServiceImpl().ruleLineChart(new SearchReq()); + System.out.println(result); } @Override public HotKeyLineChartVo ruleLineChart(SearchReq req) { - int type = req.getType(); + System.out.println("req--> "+JSON.toJSONString(req)); + int type = 1; if(req.getEndTime() == null){ req.setEndTime(new Date()); } switch (type){ case 1: req.setStartTime(DateUtil.preMinus(30)); - List list = statisticsList(); - Map map = new HashMap<>(10); - Map> listMap = list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); - for (Map.Entry> m : listMap.entrySet()) { - int start = 1; - map.put(m.getKey(),new int[30]); - int[] data = map.get(m.getKey()); - int tmp = 0; - for (int i = 0; i < 30; i++) { - Statistics st; - try { - st = m.getValue().get(tmp); - if(String.valueOf(start).endsWith("24")){ start = start + 77; } - if(start != st.getHours()){ - data[i] = 0; - }else{ - tmp ++; - data[i] = st.getCount(); - } - start++; - }catch (Exception e){ - data[i] = 0; - } - } - } - - System.out.println("30 min"); - break; + LocalDateTime startTime = DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS); + List list = statisticsList(req.getRules()); + HotKeyLineChartVo vo = CommonUtil.assembleData(list, startTime, 30,1); + System.out.println(JSON.toJSONString(vo)); + return vo; case 2: req.setStartTime(DateUtil.preDays(1)); - System.out.println("24 hours"); - break; + LocalDateTime startTime2 = DateUtil.strToLdt("20063022",DateUtil.PATTERN_HOUR); + List list2 = statisticsList1(); + HotKeyLineChartVo vo2 = CommonUtil.assembleData(list2, startTime2, 24,2); + System.out.println(JSON.toJSONString(vo2)); + return vo2; case 3: req.setStartTime(DateUtil.preDays(7)); - System.out.println("7 days"); - break; + List list3 = statisticsListDay(); + System.out.println(JSON.toJSONString(list3)); + LocalDateTime startTime3 = DateUtil.strToLdt("20062800",DateUtil.PATTERN_HOUR); + HotKeyLineChartVo vo3 = CommonUtil.assembleData(list3, startTime3, 28,2); + return vo3; default: System.out.println("============="); } - return null; + return new HotKeyLineChartVo(null,null); } @@ -317,32 +279,75 @@ public class KeyServiceImpl implements KeyService { } - private List statisticsList(){ + private static List statisticsList(String strings){ Random rd = new Random(); List list = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - Statistics st = new Statistics(); - st.setApp("rule1"); - st.setKeyName("key1"); - st.setCount(rd.nextInt(100)); - st.setBizType(1); - st.setMinutes(2006052140+i); - if(String.valueOf(st.getMinutes()).endsWith("60")){ - st.setMinutes(st.getMinutes()+1); + + if(strings.contains("428k2")){ + return list; + } + System.out.println("strings--> "+JSON.toJSONString(strings)); + + if(strings.contains("k21") && !strings.contains("k22") ){ + System.out.println("strings.contains(21)--> "+strings.contains("k21")); + + for (int i = 0; i < 30 ; i++) { + if(i == 12 || i ==13){ + }else{ + Statistics st = new Statistics(); + st.setApp("rule1"); + st.setKeyName("key1"); + st.setCount(rd.nextInt(100)); + st.setBizType(1); + // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); + // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); + int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); + // System.out.println(time); + st.setMinutes(time); + list.add(st); + } } - list.add(st); + return list; } - List list2 = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - Statistics st2 = new Statistics(); - st2.setApp("rule2"); - st2.setKeyName("key2"); - st2.setCount(rd.nextInt(100)); - st2.setBizType(1); - st2.setMinutes(2006052140+i); - list2.add(st2); + + + if(strings.contains("k21") && strings.contains("k22")){ + System.out.println("strings.contains(21 + 22)--> "+strings.contains("k21")); + + for (int i = 0; i < 30 ; i++) { + if(i == 12 || i ==13){ + }else{ + Statistics st = new Statistics(); + st.setApp("rule1"); + st.setKeyName("key1"); + st.setCount(rd.nextInt(100)); + st.setBizType(1); + // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); + // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); + int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); + // System.out.println(time); + st.setMinutes(time); + list.add(st); + } + } + List list2 = new ArrayList<>(); + for (int i = 0; i < 30 ; i++) { + if(i == 0 || i == 1 || i == 18 || i == 19 || i == 25){ + + }else{ + Statistics st2 = new Statistics(); + st2.setApp("rule2"); + st2.setKeyName("key2"); + st2.setCount(rd.nextInt(100)); + st2.setBizType(1); + st2.setMinutes(DateUtil.reviseTime(DateUtil.strToLdt("2006082355", DateUtil.PATTERN_MINUS),i,1)); + list2.add(st2); + } + } + list.addAll(list2); + return list; } - list.addAll(list2); + return list; } @@ -353,14 +358,33 @@ public class KeyServiceImpl implements KeyService { Statistics st = new Statistics(); st.setApp("rule1"); st.setKeyName("key1"); - st.setCount(rd.nextInt(100)); + st.setCount(rd.nextInt(1000)); st.setBizType(1); - st.setHours(20060500+i); + st.setHours(DateUtil.reviseTime(DateUtil.strToLdt("20063022",DateUtil.PATTERN_HOUR),i,2)); + list.add(st); } return list; } + private List statisticsListDay(){ + Random rd = new Random(); + List list = new ArrayList<>(); + for (int j = 0; j < 7; j++) { + for (int i = 0; i < 24 ; i++) { + Statistics st = new Statistics(); + st.setApp("rule"); + st.setKeyName("key"); + st.setCount(rd.nextInt(5000)); + st.setBizType(1); + st.setDays(DateUtil.reviseTime(DateUtil.strToLdt("20062800",DateUtil.PATTERN_HOUR),j,3)); + st.setHours(DateUtil.reviseTime(DateUtil.strToLdt("20062800",DateUtil.PATTERN_HOUR).plusDays(j), i,2)); + list.add(st); + } + } + return list; + } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java index 6d5a4fd..8688fba 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java @@ -100,7 +100,6 @@ public class RuleServiceImpl implements RuleService { List keyValues = configCenter.getPrefix(prefix); List rules = new ArrayList<>(); for (KeyValue kv : keyValues) { - String v = kv.getValue().toStringUtf8(); if(StringUtil.isEmpty(v)){ continue; @@ -129,4 +128,23 @@ public class RuleServiceImpl implements RuleService { configCenter.put(ConfigConstant.rulePath + app, rules.getRules()); return 1; } + + @Override + public List listRules(String app) { + List keyValues = configCenter.getPrefix(ConfigConstant.rulePath); + List rules = new ArrayList<>(); + for (KeyValue kv : keyValues) { + String v = kv.getValue().toStringUtf8(); + if(StringUtil.isEmpty(v)){ + continue; + } + String key = kv.getKey().toStringUtf8(); + String appKey = key.replace(ConfigConstant.rulePath,""); + List rs = JSON.parseArray(v, Rule.class); + for (Rule r : rs) { + rules.add(r.getKey()); + } + } + return rules; + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 42c4667..75e3809 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -1,32 +1,40 @@ package com.jd.platform.hotkey.dashboard.util; +import com.alibaba.fastjson.JSON; +import com.jd.platform.hotkey.dashboard.common.domain.vo.HotKeyLineChartVo; +import com.jd.platform.hotkey.dashboard.model.Statistics; + import java.io.UnsupportedEncodingException; -import java.util.Base64; +import java.time.LocalDateTime; +import java.util.*; +import java.util.stream.Collectors; public class CommonUtil { /** * 获取父级Key + * * @param key key * @return string */ - public static String parentK(String key){ - if(key.endsWith("/")){ - key = key.substring(0,key.length()-1); + public static String parentK(String key) { + if (key.endsWith("/")) { + key = key.substring(0, key.length() - 1); } int index = key.lastIndexOf("/"); - return key.substring(0,index+1); + return key.substring(0, index + 1); } /** * 获取AppName + * * @param k k * @return str */ - public static String appName(String k){ + public static String appName(String k) { String[] arr = k.split("/"); - for (int i = 0; i < arr.length ; i++) { - if(i == 3){ + for (int i = 0; i < arr.length; i++) { + if (i == 3) { return arr[i]; } } @@ -34,14 +42,12 @@ public class CommonUtil { } - - public static String keyName(String k){ + public static String keyName(String k) { int index = k.lastIndexOf("/"); - return k.substring(index+1); + return k.substring(index + 1); } - public static String encoder(String text) { try { return Base64.getEncoder().encodeToString(text.getBytes("utf-8")); @@ -53,9 +59,9 @@ public class CommonUtil { public static String decoder(String text) { - byte[] bytes=Base64.getDecoder().decode(text); + byte[] bytes = Base64.getDecoder().decode(text); try { - return new String(bytes,"utf-8"); + return new String(bytes, "utf-8"); } catch (UnsupportedEncodingException e) { e.printStackTrace(); } @@ -63,4 +69,72 @@ public class CommonUtil { } + /** + * 拼装数据 + * @param list list-data + * @param startTime 开始时间 + * @param size 格子数 + * @param type 类型 1分钟 2小时 + * @return vo + */ + public static HotKeyLineChartVo assembleData(List list, LocalDateTime startTime, int size, int type) { + Set set = new TreeSet<>(); + boolean isHour = type == 1; + boolean sevenDays = size == 28; + String suffix = isHour ? "60" : "24"; + String pattern = isHour ? DateUtil.PATTERN_MINUS : DateUtil.PATTERN_HOUR; + Map map = new HashMap<>(10); + Map> listMap = listGroup(list, sevenDays); + for (Map.Entry> m : listMap.entrySet()) { + int start = DateUtil.reviseTime(startTime, 0, type); + map.put(m.getKey(), new int[size]); + int[] data = map.get(m.getKey()); + int tmp = 0; + for (int i = 0; i < size; i++) { + if (String.valueOf(start).endsWith(suffix)) { + LocalDateTime tmpTime = DateUtil.strToLdt((start - 1) + "", pattern); + start = DateUtil.reviseTime(tmpTime, 1, type); + } + set.add(DateUtil.strToLdt(start + "", pattern).toString().replace("T", " ")); + Statistics st = m.getValue().get(tmp); + int val = isHour ? st.getMinutes() : st.getHours(); + if (start != val) { + data[i] = 0; + } else { + tmp++; + data[i] = st.getCount(); + } + if (sevenDays) { + start += 6; + } else { + start++; + } + + } + } + System.out.println("-------"); + + System.out.println(JSON.toJSONString(new HotKeyLineChartVo(new ArrayList<>(set), map))); + return new HotKeyLineChartVo(new ArrayList<>(set), map); + } + + + /** + * 分组 + * @param list list + * @param sevenDays 是否7天 + * @return map + */ + private static Map> listGroup(List list, boolean sevenDays){ + return list.stream().filter(x -> { + if(sevenDays){ + String hs = String.valueOf(x.getHours()); + return Integer.parseInt(hs.substring(hs.length() - 2)) % 6 == 0; + }else{ + return true; + } + }).collect(Collectors.groupingBy(Statistics::getKeyName)); + } + + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java index 8864fe6..401e6fe 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java @@ -2,6 +2,7 @@ package com.jd.platform.hotkey.dashboard.util; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; @@ -13,13 +14,20 @@ import java.util.Date; public class DateUtil { - private static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + public static final String PATTERN_MINUS="yyMMddHHmm"; + + public static final String PATTERN_HOUR="yyMMddHH"; + + public static final String PATTERN_DAY="yyMMdd"; - public static final DateTimeFormatter TIME_FORMAT1 = DateTimeFormatter.ofPattern("yyMMddHHmm"); + private static final DateTimeFormatter FORMAT_MINUS = DateTimeFormatter.ofPattern(PATTERN_MINUS); - public static final DateTimeFormatter TIME_FORMAT2 = DateTimeFormatter.ofPattern("yyMMddHH"); + private static final DateTimeFormatter FORMAT_HOUR = DateTimeFormatter.ofPattern(PATTERN_HOUR); + + private static final DateTimeFormatter FORMAT_DAY = DateTimeFormatter.ofPattern(PATTERN_DAY); + + private static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); - public static final DateTimeFormatter TIME_FORMAT3 = DateTimeFormatter.ofPattern("yyMMddHH"); public static Date strToDate(String str){ try { @@ -30,37 +38,62 @@ public class DateUtil { return null; } + public static LocalDateTime strToLdt(String str, String pattern){ + return LocalDateTime.parse(str, DateTimeFormatter.ofPattern(pattern)); + } + - public static Date localDateTimeToDate(LocalDateTime localDateTime){ + public static int reviseTime(LocalDateTime time, int diff, int type){ + switch (type){ + case 1: + return Integer.parseInt(FORMAT_MINUS.format(time.plusMinutes(diff))); + case 2: + return Integer.parseInt(FORMAT_HOUR.format(time.plusHours(diff))); + case 3: + return Integer.parseInt(FORMAT_DAY.format(time.plusDays(diff))); + default: + } + return 0; + } + + + + + public static Date ldtToDate(LocalDateTime localDateTime){ return Date.from(localDateTime.atZone(ZoneId.systemDefault()).toInstant()); } + public static LocalDateTime dateToLdt(Date date){ + return LocalDateTime.ofInstant( date.toInstant(), ZoneId.systemDefault()); + } + + public static int nowMinus(LocalDateTime now){ - return Integer.parseInt(now.format(TIME_FORMAT1)) ; + return Integer.parseInt(now.format(FORMAT_MINUS)) ; } public static int nowHour(LocalDateTime now){ - return Integer.parseInt(now.format(TIME_FORMAT2)); + return Integer.parseInt(now.format(FORMAT_HOUR)); } - public static int nowDay(LocalDateTime now){ return Integer.parseInt(now.format(TIME_FORMAT3));} + public static int nowDay(LocalDateTime now){ return Integer.parseInt(now.format(FORMAT_DAY));} public static int preHours(LocalDateTime now, int hours){ - return Integer.parseInt(now.minusHours(hours).format(TIME_FORMAT2)); + return Integer.parseInt(now.minusHours(hours).format(FORMAT_HOUR)); } public static Date preTime(int hours){ - return localDateTimeToDate(LocalDateTime.now().minusHours(hours)); + return ldtToDate(LocalDateTime.now().minusHours(hours)); } public static Date preMinus(int minus){ - return localDateTimeToDate(LocalDateTime.now().minusMinutes(minus)); + return ldtToDate(LocalDateTime.now().minusMinutes(minus)); } public static Date preDays(int days){ - return localDateTimeToDate(LocalDateTime.now().minusDays(days)); + return ldtToDate(LocalDateTime.now().minusDays(days)); } } diff --git a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js index 9ced405..ebc6aaa 100644 --- a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js +++ b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js @@ -495,8 +495,8 @@ modal_status = { function getCookie(cname){ let token = window.localStorage.getItem('token'); let time = window.localStorage.getItem('time'); - console.log("token---> "+token); - console.log("存入的时间---> "+time+" 当前的时间---> "+Date.now()); + // console.log("token---> "+token); + // console.log("存入的时间---> "+time+" 当前的时间---> "+Date.now()); if(token != null && token !==""){ let time = window.localStorage.getItem('time'); if(Date.now()-time>7*24*360000){ diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index 96da0de..8f48754 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -1,29 +1,97 @@ - - + + + + +
-
-
hotKey
-
+
+
+
+
+
+
+ + + + + + + + + + + + + + +
+ + + +
+
+
+
+
+
+
+
+
+
+
+
-
+ + + diff --git a/dashboard/src/main/resources/templates/admin/mainbak.html b/dashboard/src/main/resources/templates/admin/mainbak.html new file mode 100644 index 0000000..96da0de --- /dev/null +++ b/dashboard/src/main/resources/templates/admin/mainbak.html @@ -0,0 +1,106 @@ + + + + + + + +
+
+
hotKey
+
+ +
+
+ + + + + + + + + + + -- Gitee From d393a336fc2d3bfb035db95511524e6772a63b02 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 12 Jun 2020 17:11:09 +0800 Subject: [PATCH 02/51] =?UTF-8?q?=E8=AE=BE=E7=BD=AEnetty=E7=9A=84worker?= =?UTF-8?q?=E7=9A=84=E7=BA=BF=E7=A8=8B=E6=95=B0=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/worker/cache/CaffeineBuilder.java | 22 ++++++++++++++++--- .../platform/hotkey/worker/tool/CpuNum.java | 10 ++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java index 6c60f17..8eb1b9f 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java @@ -3,6 +3,8 @@ package com.jd.platform.hotkey.worker.cache; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; /** @@ -10,6 +12,7 @@ import java.util.concurrent.TimeUnit; * @version 1.0 */ public class CaffeineBuilder { + private static ExecutorService executorService = Executors.newFixedThreadPool(4); /** * 构建所有来的要缓存的key cache */ @@ -17,8 +20,9 @@ public class CaffeineBuilder { //老版本jdk1.8.0_20之前，caffeine默认的forkJoinPool在及其密集的淘汰过期时，会有forkJoinPool报错。建议用新版jdk return Caffeine.newBuilder() .initialCapacity(8192)//初始大小 - .maximumSize(5000000)//最大数量 + .maximumSize(20000000)//最大数量 .expireAfterWrite(1, TimeUnit.MINUTES)//过期时间 + .executor(executorService) .softValues() .build(); } @@ -29,14 +33,26 @@ public class CaffeineBuilder { public static Cache buildRecentHotKeyCache() { return Caffeine.newBuilder() .initialCapacity(256)//初始大小 - .maximumSize(50000)//最大数量 + .maximumSize(500000)//最大数量 .expireAfterWrite(10, TimeUnit.SECONDS)//过期时间 + .executor(executorService) .softValues() .build(); } public static void main(String[] args) { - System.out.println(System.getProperty("java.version")); +// Cache cache = buildAllKeyCache(); +// //开启上传worker信息 +// ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); +// scheduledExecutorService.scheduleAtFixedRate(() -> { +// System.out.println(cache.asMap().size()); +// +// }, 0, 5, TimeUnit.SECONDS); +// +// long i = 0; +// while (true) { +// cache.put(i++ + "", i); +// } } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java index c886bfe..4c9a735 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java @@ -11,11 +11,15 @@ public class CpuNum { * netty worker线程数量. cpu密集型 */ public static int workerCount() { + int count = 1; if (isNewerVersion()) { - return Runtime.getRuntime().availableProcessors(); - } else { - return 4; + count = Runtime.getRuntime().availableProcessors(); } + if (count >= 4) { + count = count / 2; + } + + return count; } public static void main(String[] args) { -- Gitee From 6ab3566c6e7728a85d3de6330ca9d0035a19eef9 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Mon, 15 Jun 2020 20:25:21 +0800 Subject: [PATCH 03/51] update README.md. --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index c557dae..772a242 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,11 @@ ![输入图片说明](https://images.gitee.com/uploads/images/2020/0611/152249_4ac01178_303698.png "屏幕截图.png") +### 微信群 +![输入图片说明](https://images.gitee.com/uploads/images/2020/0615/202512_3a190552_303698.jpeg "WechatIMG3.jpeg") + + + #### 安装教程 1. xxxx -- Gitee From e6a11b9d711b1e7274b156b62254ef88e4a6cd09 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 16 Jun 2020 01:51:58 +0800 Subject: [PATCH 04/51] =?UTF-8?q?=E9=A6=96=E9=A1=B5=E5=A4=A7=E6=94=B91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/domain/req/SearchReq.java | 10 - .../dashboard/common/monitor/DataHandler.java | 65 ++-- .../dashboard/mapper/StatisticsMapper.java | 15 +- .../service/impl/KeyServiceImpl.java | 305 ++++++++++-------- .../hotkey/dashboard/util/CommonUtil.java | 26 +- .../hotkey/dashboard/util/DateUtil.java | 8 +- dashboard/src/main/resources/application.yml | 2 +- .../dashboard/mapper/KeyRecordMapper.xml | 9 +- .../dashboard/mapper/StatisticsMapper.xml | 29 +- .../templates/admin/common/html/leftMenu.html | 5 +- .../main/resources/templates/admin/main.html | 284 +++++++++------- .../resources/templates/admin/mainbak.html | 76 +++++ .../resources/templates/admin/user/edit.html | 2 +- 13 files changed, 524 insertions(+), 312 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java index 57b7f0a..7bb37ef 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java @@ -31,8 +31,6 @@ public class SearchReq implements Serializable { private String key; - private String rules; - public SearchReq() { } @@ -84,14 +82,6 @@ public class SearchReq implements Serializable { this.key = key; } - public String getRules() { - return rules; - } - - public void setRules(String rules) { - this.rules = rules; - } - public SearchReq(LocalDateTime st) { this.startTime = DateUtil.ldtToDate(st); this.endTime = new Date(); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 4308467..edc1a0e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -117,42 +117,56 @@ public class DataHandler { //手工添加的是时间戳13位，worker传过来的是uuid String source = v.length() == 13 ? Constant.HAND : Constant.SYSTEM; timelyKeyRecordTwoTuple.setFirst(new KeyTimely(arr[1], v, arr[0], ttl, uuid, date)); - KeyRecord keyRecord = new KeyRecord(arr[1], v, arr[0], ttl, source, type, uuid, date); - keyRecord.setRule(RuleUtil.rule(appKey)); + + // 这是一个骚操作 在线上不方便新增rule字段的时候 临时用下val + String rule = RuleUtil.rule(appKey); + KeyRecord keyRecord = new KeyRecord(arr[1], rule, arr[0], ttl, source, type, uuid, date); + keyRecord.setRule(rule); timelyKeyRecordTwoTuple.setSecond(keyRecord); return timelyKeyRecordTwoTuple; } else if (eventType.equals(Event.EventType.DELETE)) { timelyKeyRecordTwoTuple.setFirst(new KeyTimely(arr[1], null, arr[0], 0L, null, null)); - //删除事件就不记录了 -// timelyKeyRecordTwoTuple.setSecond(new KeyRecord(arr[1], v, arr[0], 0L, Constant.SYSTEM, type, uuid, date)); return timelyKeyRecordTwoTuple; } return timelyKeyRecordTwoTuple; } + + /** + * 每小时 统计一次record 表 结果记录到统计表 + */ @Scheduled(cron = "0 0 * * * ?") public void offlineStatistics() { try { - // 每小时 统计一次record 表 结果记录到统计表 LocalDateTime now = LocalDateTime.now(); Date nowTime = DateUtil.ldtToDate(now); int day = DateUtil.nowDay(now); int hour = DateUtil.nowHour(now); - - List records = keyRecordMapper.maxHotKey(new SearchReq(now.minusHours(1))); - if (records.size() == 0) { - return; + SearchReq preHour = new SearchReq(now.minusHours(1)); + List records = keyRecordMapper.maxHotKey(preHour); + if (records.size() != 0) { + records.forEach(x -> { + x.setBizType(1); + x.setCreateTime(nowTime); + x.setDays(day); + x.setHours(hour); + x.setUuid(1 + "_" + x.getKeyName() + "_" + hour); + }); + } + List statistics = keyRecordMapper.statisticsByRule(preHour); + if (statistics.size() != 0) { + statistics.forEach(x -> { + x.setBizType(6); + x.setCreateTime(nowTime); + x.setDays(day); + x.setHours(hour); + x.setUuid(6 + "_" + x.getKeyName() + "_" + hour); + }); + records.addAll(statistics); } - records.forEach(x -> { - x.setBizType(1); - x.setCreateTime(nowTime); - x.setDays(day); - x.setHours(hour); - x.setUuid(1 + "_" + x.getKeyName() + "_" + hour); - }); int row = statisticsMapper.batchInsert(records); - log.info("定时统计热点记录时间：{}, 影响行数：{}", now.toString(), row); + log.info("每小时统计最热点和规则，时间：{}, 影响行数：{}", now.toString(), row); } catch (Exception e) { e.printStackTrace(); } @@ -160,28 +174,33 @@ public class DataHandler { } - //@Scheduled(cron = "0 0 * * * ?") + /** + * 每分钟统计一次record 表 结果记录到统计表 + */ + // @Scheduled(cron = "0 */1 * * * ?") public void offlineStatisticsRule() { try { - // 每分钟小时 统计一次record 表 结果记录到统计表 LocalDateTime now = LocalDateTime.now(); Date nowTime = DateUtil.ldtToDate(now); int day = DateUtil.nowDay(now); int hour = DateUtil.nowHour(now); + int minus = DateUtil.nowMinus(now); - List records = keyRecordMapper.statisticsByRule(null); + List records = keyRecordMapper.statisticsByRule(new SearchReq(now.minusMinutes(1))); if (records.size() == 0) { return; } records.forEach(x -> { - x.setBizType(1); + x.setBizType(5); x.setCreateTime(nowTime); x.setDays(day); x.setHours(hour); - x.setUuid(1 + "_" + x.getKeyName() + "_" + hour); + x.setMinutes(minus); + // 骚操作 临时解决没有rule字段的问题 + x.setUuid(5 + "_" + x.getKeyName() + "_" + minus); }); int row = statisticsMapper.batchInsert(records); - log.info("定时统计热点记录时间：{}, 影响行数：{}", now.toString(), row); + log.info("每分钟统计规则，时间：{}, 影响行数：{}", now.toString(), row); } catch (Exception e) { e.printStackTrace(); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java index 3c2ee08..fc5c79d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java @@ -18,7 +18,14 @@ public interface StatisticsMapper { * 查看 * @return list */ - List listStatistics(ChartReq chartReq); + List listStatistics(SearchReq req); + + + /** + * 查看 + * @return list + */ + List listOrderByTime(SearchReq req); /** * records @@ -33,4 +40,10 @@ public interface StatisticsMapper { * @return list */ List sumStatistics(SearchReq req); + + /** + * 清理 + * @param type type + */ + void clear(int type); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 2bf3a06..d32e630 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -53,45 +53,34 @@ public class KeyServiceImpl implements KeyService { @Resource private StatisticsMapper statisticsMapper; - public static void main(String[] args) { - HotKeyLineChartVo result = new KeyServiceImpl().ruleLineChart(new SearchReq()); - System.out.println(result); - } - + /** + * 折线图 + * @param req req + * @return vo + */ @Override public HotKeyLineChartVo ruleLineChart(SearchReq req) { - System.out.println("req--> "+JSON.toJSONString(req)); - int type = 1; - if(req.getEndTime() == null){ - req.setEndTime(new Date()); + int type = req.getType(); + LocalDateTime now = LocalDateTime.now(); + req.setEndTime(req.getEndTime() == null ? DateUtil.ldtToDate(now) : req.getEndTime()); + if(type == 5){ + LocalDateTime startTime = now.minusMinutes(31); + req.setStartTime(DateUtil.ldtToDate(startTime)); + List list = statisticsMapper.listOrderByTime(req); + return CommonUtil.assembleData(list, startTime, 30,1); + }else if(type == 6){ + LocalDateTime startTime2 = now.minusHours(25); + req.setStartTime(DateUtil.ldtToDate(startTime2)); + List list2 = statisticsMapper.listOrderByTime(req); + return CommonUtil.assembleData(list2, startTime2, 24,2); + }else{ + LocalDateTime startTime3 = now.minusDays(7).minusHours(1); + req.setStartTime(DateUtil.ldtToDate(startTime3)); + req.setType(6); + List list3 = statisticsMapper.listOrderByTime(req); + return CommonUtil.assembleData(list3, startTime3, 7*24,2); } - switch (type){ - case 1: - req.setStartTime(DateUtil.preMinus(30)); - LocalDateTime startTime = DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS); - List list = statisticsList(req.getRules()); - HotKeyLineChartVo vo = CommonUtil.assembleData(list, startTime, 30,1); - System.out.println(JSON.toJSONString(vo)); - return vo; - case 2: - req.setStartTime(DateUtil.preDays(1)); - LocalDateTime startTime2 = DateUtil.strToLdt("20063022",DateUtil.PATTERN_HOUR); - List list2 = statisticsList1(); - HotKeyLineChartVo vo2 = CommonUtil.assembleData(list2, startTime2, 24,2); - System.out.println(JSON.toJSONString(vo2)); - return vo2; - case 3: - req.setStartTime(DateUtil.preDays(7)); - List list3 = statisticsListDay(); - System.out.println(JSON.toJSONString(list3)); - LocalDateTime startTime3 = DateUtil.strToLdt("20062800",DateUtil.PATTERN_HOUR); - HotKeyLineChartVo vo3 = CommonUtil.assembleData(list3, startTime3, 28,2); - return vo3; - default: - System.out.println("============="); - } - return new HotKeyLineChartVo(null,null); } @@ -125,12 +114,10 @@ public class KeyServiceImpl implements KeyService { public HotKeyLineChartVo getLineChart(ChartReq chartReq) { int hours = 6; // 默认查询6小时内的数据 - if(chartReq.getStartTime() == null || chartReq.getEndTime() == null){ - chartReq.setStartTime(DateUtil.preTime(hours)); - chartReq.setEndTime(new Date()); - } - - List statistics = statisticsMapper.listStatistics(chartReq); + SearchReq req = new SearchReq(); + req.setStartTime(DateUtil.preTime(hours)); + req.setEndTime(new Date()); + List statistics = statisticsMapper.listStatistics(req); // 获取data Y轴 Map keyDateMap = keyDateMap(statistics, hours); // 获取时间x轴 @@ -241,7 +228,7 @@ public class KeyServiceImpl implements KeyService { Map map = new HashMap<>(10); Map> listMap = statistics.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); for (Map.Entry> m : listMap.entrySet()) { - int start = DateUtil.preHours(LocalDateTime.now(),5); + int start = DateUtil.preHoursInt(5); map.put(m.getKey(),new int[hours]); int[] data = map.get(m.getKey()); int tmp = 0; @@ -279,112 +266,166 @@ public class KeyServiceImpl implements KeyService { } - private static List statisticsList(String strings){ + private List statisticsList(){ Random rd = new Random(); - List list = new ArrayList<>(); - - if(strings.contains("428k2")){ - return list; + List records = new ArrayList<>(); + statisticsMapper.clear(1); + for (int i = 0; i < 35; i++) { + // 每分钟小时 统计一次record 表 结果记录到统计表 + LocalDateTime now = LocalDateTime.now().minusMinutes(35-i); + Date nowTime = DateUtil.ldtToDate(now); + int day = DateUtil.nowDay(now); + int hour = DateUtil.nowHour(now); + int minus = DateUtil.nowMinus(now); + + Statistics s1 = new Statistics(); + s1.setKeyName("k21"); + s1.setApp("key21APP"); + s1.setCreateTime(nowTime); + s1.setDays(day); + s1.setHours(hour); + s1.setCount(rd.nextInt(800)); + s1.setBizType(1); + s1.setMinutes(minus); + s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); + records.add(s1); + + Statistics s11 = new Statistics(); + s11.setKeyName("k22"); + s11.setApp("key22APP"); + s11.setBizType(1); + s11.setCreateTime(nowTime); + s11.setDays(day); + s11.setMinutes(minus); + s11.setCount(rd.nextInt(600)); + s11.setHours(hour); + s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); + records.add(s11); } - System.out.println("strings--> "+JSON.toJSONString(strings)); - - if(strings.contains("k21") && !strings.contains("k22") ){ - System.out.println("strings.contains(21)--> "+strings.contains("k21")); - - for (int i = 0; i < 30 ; i++) { - if(i == 12 || i ==13){ - }else{ - Statistics st = new Statistics(); - st.setApp("rule1"); - st.setKeyName("key1"); - st.setCount(rd.nextInt(100)); - st.setBizType(1); - // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); - // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); - int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); - // System.out.println(time); - st.setMinutes(time); - list.add(st); - } + statisticsMapper.batchInsert(records); + List list = new ArrayList<>(); + for (int i = 0; i < 30 ; i++) { + if(i == 12 || i ==13){ + }else{ + Statistics st = new Statistics(); + st.setApp("k21"); + st.setKeyName("k21"); + st.setCount(rd.nextInt(100)); + st.setBizType(1); + // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); + // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); + int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); + // System.out.println(time); + st.setMinutes(time); + list.add(st); } - return list; } - - - if(strings.contains("k21") && strings.contains("k22")){ - System.out.println("strings.contains(21 + 22)--> "+strings.contains("k21")); - - for (int i = 0; i < 30 ; i++) { - if(i == 12 || i ==13){ - }else{ - Statistics st = new Statistics(); - st.setApp("rule1"); - st.setKeyName("key1"); - st.setCount(rd.nextInt(100)); - st.setBizType(1); - // LocalDateTime ldf = DateUtil.strToLdt("2006082355"); - // System.out.println(ldf.toString()+" - "+DateUtil.strToLdt("2006082355")); - int time = DateUtil.reviseTime(DateUtil.strToLdt("2006082355",DateUtil.PATTERN_MINUS), i, 1); - // System.out.println(time); - st.setMinutes(time); - list.add(st); - } - } - List list2 = new ArrayList<>(); - for (int i = 0; i < 30 ; i++) { - if(i == 0 || i == 1 || i == 18 || i == 19 || i == 25){ - - }else{ - Statistics st2 = new Statistics(); - st2.setApp("rule2"); - st2.setKeyName("key2"); - st2.setCount(rd.nextInt(100)); - st2.setBizType(1); - st2.setMinutes(DateUtil.reviseTime(DateUtil.strToLdt("2006082355", DateUtil.PATTERN_MINUS),i,1)); - list2.add(st2); - } + List list2 = new ArrayList<>(); + for (int i = 0; i < 30 ; i++) { + if(i == 0 || i == 1 || i == 18 || i == 19 || i == 25){ + + }else{ + Statistics st2 = new Statistics(); + st2.setApp("k22"); + st2.setKeyName("k22"); + st2.setCount(rd.nextInt(100)); + st2.setBizType(1); + st2.setMinutes(DateUtil.reviseTime(DateUtil.strToLdt("2006082355", DateUtil.PATTERN_MINUS),i,1)); + list2.add(st2); } - list.addAll(list2); - return list; } - + list.addAll(list2); return list; + } - private List statisticsList1(){ + private List statisticsList1(int type){ Random rd = new Random(); - List list = new ArrayList<>(); - for (int i = 0; i < 24 ; i++) { - Statistics st = new Statistics(); - st.setApp("rule1"); - st.setKeyName("key1"); - st.setCount(rd.nextInt(1000)); - st.setBizType(1); - st.setHours(DateUtil.reviseTime(DateUtil.strToLdt("20063022",DateUtil.PATTERN_HOUR),i,2)); - list.add(st); - } - return list; - } + List records = new ArrayList<>(); + statisticsMapper.clear(6); + + if(type == 3){ + int temp = 1; + for (int j = 0; j < 9; j++) { + for (int i = 0; i < 24; i++) { + // 每分钟小时 统计一次record 表 结果记录到统计表 + LocalDateTime now = LocalDateTime.now().minusHours(9 * 24 - temp); + + Date nowTime = DateUtil.ldtToDate(now); + int day = DateUtil.nowDay(now); + int hour = DateUtil.nowHour(now); + Statistics s1 = new Statistics(); + s1.setKeyName("k21"); + s1.setApp("key21APP"); + s1.setCreateTime(nowTime); + s1.setDays(day); + s1.setHours(hour); + s1.setCount(rd.nextInt(200)); + s1.setBizType(6); + s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); + records.add(s1); + + Statistics s11 = new Statistics(); + s11.setKeyName("k22"); + s11.setApp("key22APP"); + s11.setBizType(6); + s11.setCreateTime(nowTime); + s11.setDays(day); + s11.setCount(rd.nextInt(500)); + s11.setHours(hour); + s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); + records.add(s11); + + Statistics s33 = new Statistics(); + s33.setKeyName("k33"); + s33.setApp("key33APP"); + s33.setBizType(6 ); + s33.setCreateTime(nowTime); + s33.setDays(day); + s33.setCount(rd.nextInt(300)); + s33.setHours(hour); + s33.setUuid(3 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); + records.add(s33); + temp++; + } + } - private List statisticsListDay(){ - Random rd = new Random(); - List list = new ArrayList<>(); - for (int j = 0; j < 7; j++) { - for (int i = 0; i < 24 ; i++) { - Statistics st = new Statistics(); - st.setApp("rule"); - st.setKeyName("key"); - st.setCount(rd.nextInt(5000)); - st.setBizType(1); - st.setDays(DateUtil.reviseTime(DateUtil.strToLdt("20062800",DateUtil.PATTERN_HOUR),j,3)); - st.setHours(DateUtil.reviseTime(DateUtil.strToLdt("20062800",DateUtil.PATTERN_HOUR).plusDays(j), i,2)); - list.add(st); + }else{ + for (int i = 0; i < 30; i++) { + // 每分钟小时 统计一次record 表 结果记录到统计表 + LocalDateTime now = LocalDateTime.now().minusHours(30-i); + Date nowTime = DateUtil.ldtToDate(now); + int day = DateUtil.nowDay(now); + int hour = DateUtil.nowHour(now); + Statistics s1 = new Statistics(); + s1.setKeyName("key21"); + s1.setApp("key21APP"); + s1.setCreateTime(nowTime); + s1.setDays(day); + s1.setHours(hour); + s1.setCount(rd.nextInt(800)); + s1.setBizType(2); + s1.setUuid(1 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); + records.add(s1); + + Statistics s11 = new Statistics(); + s11.setKeyName("key22"); + s11.setApp("key22APP"); + s11.setBizType(2); + s11.setCreateTime(nowTime); + s11.setDays(day); + s11.setCount(rd.nextInt(600)); + s11.setHours(hour); + s11.setUuid(2 + "_" + s1.getKeyName() + "_" + hour+ UUID.randomUUID().toString()); + records.add(s11); } } - return list; + statisticsMapper.batchInsert(records); + return records; } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index 75e3809..6cbd81c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -80,11 +80,11 @@ public class CommonUtil { public static HotKeyLineChartVo assembleData(List list, LocalDateTime startTime, int size, int type) { Set set = new TreeSet<>(); boolean isHour = type == 1; - boolean sevenDays = size == 28; String suffix = isHour ? "60" : "24"; String pattern = isHour ? DateUtil.PATTERN_MINUS : DateUtil.PATTERN_HOUR; Map map = new HashMap<>(10); - Map> listMap = listGroup(list, sevenDays); + Map> listMap = listGroup(list); + for (Map.Entry> m : listMap.entrySet()) { int start = DateUtil.reviseTime(startTime, 0, type); map.put(m.getKey(), new int[size]); @@ -98,22 +98,16 @@ public class CommonUtil { set.add(DateUtil.strToLdt(start + "", pattern).toString().replace("T", " ")); Statistics st = m.getValue().get(tmp); int val = isHour ? st.getMinutes() : st.getHours(); + System.out.println("start--> "+start+" val---> "+val+" tmp---> "+tmp); if (start != val) { data[i] = 0; } else { tmp++; data[i] = st.getCount(); } - if (sevenDays) { - start += 6; - } else { - start++; - } - + start++; } } - System.out.println("-------"); - System.out.println(JSON.toJSONString(new HotKeyLineChartVo(new ArrayList<>(set), map))); return new HotKeyLineChartVo(new ArrayList<>(set), map); } @@ -122,18 +116,10 @@ public class CommonUtil { /** * 分组 * @param list list - * @param sevenDays 是否7天 * @return map */ - private static Map> listGroup(List list, boolean sevenDays){ - return list.stream().filter(x -> { - if(sevenDays){ - String hs = String.valueOf(x.getHours()); - return Integer.parseInt(hs.substring(hs.length() - 2)) % 6 == 0; - }else{ - return true; - } - }).collect(Collectors.groupingBy(Statistics::getKeyName)); + private static Map> listGroup(List list){ + return list.stream().collect(Collectors.groupingBy(Statistics::getKeyName)); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java index 401e6fe..bb49094 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java @@ -80,8 +80,12 @@ public class DateUtil { public static int nowDay(LocalDateTime now){ return Integer.parseInt(now.format(FORMAT_DAY));} - public static int preHours(LocalDateTime now, int hours){ - return Integer.parseInt(now.minusHours(hours).format(FORMAT_HOUR)); + public static int preHoursInt(int hours){ + return Integer.parseInt(LocalDateTime.now().minusHours(hours).format(FORMAT_HOUR)); + } + + public static Date preHours(int hours){ + return ldtToDate(LocalDateTime.now().minusHours(hours)); } public static Date preTime(int hours){ diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index 9e852a2..b910068 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -29,7 +29,7 @@ spring : datasource: username: root password: JRTEST - url: jdbc:mysql://172.24.7.182:3306/test1?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC + url: jdbc:mysql://172.24.7.182:3306/test1?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT%2B8 driver-class-name: com.mysql.cj.jdbc.Driver pagehelper: helperDialect: mysql diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml index cba7852..a18ac6d 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRecordMapper.xml @@ -184,8 +184,8 @@ diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml index 1435472..fd34607 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.xml @@ -14,9 +14,28 @@ - select id, key_name, count, app, days, hours, minutes, biz_type, uuid, create_time - from hk_statistics order by count desc + from hk_statistics + where 1=1 + + = #{startTime} and create_time < #{endTime}]]> + + order by count desc + + + + @@ -53,4 +72,10 @@ + + \ No newline at end of file diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index fa5ae9b..b1aea2e 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -21,8 +21,9 @@
  • - - 系统设置 + + 监控大盘
    • diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index 8f48754..d72d0a2 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -14,40 +14,50 @@
    -
    - - - - - - - - - - - - - - + +
    + + + +
    + +
    -
    -
    -
    +
    +
    @@ -62,78 +72,134 @@ + + + diff --git a/dashboard/src/main/resources/templates/admin/user/edit.html b/dashboard/src/main/resources/templates/admin/user/edit.html index f04a63b..a4bb70d 100644 --- a/dashboard/src/main/resources/templates/admin/user/edit.html +++ b/dashboard/src/main/resources/templates/admin/user/edit.html @@ -48,7 +48,7 @@
    - +
    -- Gitee From 740e595bf37126061b60d0c01796e1cab3d321e2 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 16 Jun 2020 10:07:49 +0800 Subject: [PATCH 05/51] =?UTF-8?q?=E5=88=A4=E6=96=ADJava=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E8=8E=B7=E5=8F=96docker=E5=86=85cpu=E6=A0=B8=E6=95=B0=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java index 4c9a735..0b8dd10 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/CpuNum.java @@ -31,7 +31,7 @@ public class CpuNum { try { //如1.8.0_20, 1.8.0_181,1.8.0_191-b12 String javaVersion = System.getProperty("java.version"); - //1.8.0_131之前的java版本，在docker内获取availableProcessors的数量都不对，会取到宿主机的cpu数量，譬如宿主机32核， + //1.8.0_191之前的java版本，在docker内获取availableProcessors的数量都不对，会取到宿主机的cpu数量，譬如宿主机32核， //该docker只分配了4核，那么老版会取到32，新版会取到4。 //线上事故警告！！！！！！老版jdk取到数值过大，线程数太多，导致cpu瞬间100%，大量的线程切换等待 //先取前三位进行比较 @@ -49,7 +49,7 @@ public class CpuNum { smallVersion = smallVersion.substring(0, smallVersion.indexOf("-")); } - return Integer.valueOf(smallVersion) >= 131; + return Integer.valueOf(smallVersion) >= 191; } } catch (Exception e) { return false; -- Gitee From 95adf8ef0bd13d6c69b012de9c8f4dcc10101898 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 16 Jun 2020 11:01:08 +0800 Subject: [PATCH 06/51] =?UTF-8?q?=E6=89=93=E5=BC=80=E5=AE=9A=E6=97=B6?= =?UTF-8?q?=E5=92=8C=E7=9B=91=E6=8E=A7=E7=9A=84=E6=B3=A8=E8=A7=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/common/monitor/DataHandler.java | 2 +- .../platform/hotkey/dashboard/common/monitor/EtcdMonitor.java | 2 +- .../platform/hotkey/dashboard/interceptor/JwtInterceptor.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index edc1a0e..bc94342 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -177,7 +177,7 @@ public class DataHandler { /** * 每分钟统计一次record 表 结果记录到统计表 */ - // @Scheduled(cron = "0 */1 * * * ?") + @Scheduled(cron = "0 */1 * * * ?") public void offlineStatisticsRule() { try { LocalDateTime now = LocalDateTime.now(); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java index 680d6d5..26e5d39 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java @@ -36,7 +36,7 @@ import java.util.concurrent.CompletableFuture; * @Date: 2020/4/18 18:29 */ @SuppressWarnings("ALL") -//@Component +@Component public class EtcdMonitor { private static Logger log = LoggerFactory.getLogger(EtcdMonitor.class); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java index 9f3fe55..65228ac 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java @@ -45,7 +45,7 @@ public class JwtInterceptor extends HandlerInterceptorAdapter{ return true; } // appUser只读 - if(url.contains("view")||url.contains("list")){ + if(url.contains("view")||url.contains("list")||url.contains("info")){ return true; } throw new BizException(ResultEnum.NO_PERMISSION); -- Gitee From 07a65c55bc12ad4ea1febdbe6606fc65940a7cd7 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Tue, 16 Jun 2020 11:08:02 +0800 Subject: [PATCH 07/51] update README.md. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 772a242..cefde34 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # hotkey +![输入图片说明](https://images.gitee.com/uploads/images/2020/0616/105737_e5b876cd_303698.png "redis热key探测及缓存到JVM (1).png") 正在京东APP后台灰度了几千台机器，等618海量并发检验后再推广使用，暂时不要下载使用它。 -- Gitee From 0285f6ec60c6db92aca90e32328ea07ace04b92f Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 16 Jun 2020 12:51:31 +0800 Subject: [PATCH 08/51] =?UTF-8?q?=E7=AE=A1=E7=90=86=E5=85=83=E4=B8=8D?= =?UTF-8?q?=E8=83=BD=E4=BF=AE=E6=94=B9=E8=87=AA=E5=B7=B1=20=20=E8=83=BD?= =?UTF-8?q?=E7=BC=96=E8=BE=91=E5=88=AB=E4=BA=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/resources/templates/admin/user/edit.html | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dashboard/src/main/resources/templates/admin/user/edit.html b/dashboard/src/main/resources/templates/admin/user/edit.html index a4bb70d..5b71841 100644 --- a/dashboard/src/main/resources/templates/admin/user/edit.html +++ b/dashboard/src/main/resources/templates/admin/user/edit.html @@ -45,21 +45,21 @@
-
+
- +
-
+
-- Gitee From e8e80c6847387e8de7deedd1c6ce7548438eb510 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 17 Jun 2020 12:12:23 +0800 Subject: [PATCH 09/51] =?UTF-8?q?=E5=88=A4=E6=96=ADJava=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E8=8E=B7=E5=8F=96docker=E5=86=85cpu=E6=A0=B8=E6=95=B0=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dashboard/src/main/resources/templates/admin/main.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index d72d0a2..5e60d90 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -169,7 +169,7 @@ }; //加载数据 initLegend(option); - loadData(option, "", {"type":1}); + loadData(option, "", {"type":5}); // 使用刚指定的配置项和数据显示图表。 myChart.setOption(option); -- Gitee From b61d4acab26ade151b8236e82247def7adb4e906 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Wed, 17 Jun 2020 13:37:43 +0800 Subject: [PATCH 10/51] =?UTF-8?q?=E4=BF=AE=E6=94=B9etcd=E8=BF=94=E5=9B=9Er?= =?UTF-8?q?ule=20=E4=BD=BF=E4=B9=8B=E5=92=8CruleUtil=E5=AD=98=E5=85=A5?= =?UTF-8?q?=E7=9A=84=E6=A0=BC=E5=BC=8F=E4=B8=80=E8=87=B4=E7=94=A8=E4=BA=8E?= =?UTF-8?q?=E5=B1=95=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../platform/hotkey/dashboard/service/impl/RuleServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java index 8688fba..624d706 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java @@ -142,7 +142,7 @@ public class RuleServiceImpl implements RuleService { String appKey = key.replace(ConfigConstant.rulePath,""); List rs = JSON.parseArray(v, Rule.class); for (Rule r : rs) { - rules.add(r.getKey()); + rules.add(appKey+"-"+r.getKey()); } } return rules; -- Gitee From ad22bdbe59aad71fc5f6d01e91e1eaba0ab65cd1 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 17 Jun 2020 14:44:55 +0800 Subject: [PATCH 11/51] 1 --- .../common/domain/req/SearchReq.java | 1 - .../dashboard/common/monitor/DataHandler.java | 1 - .../dashboard/controller/RuleController.java | 3 +-- .../dashboard/controller/UserController.java | 11 +++++----- .../dashboard/mapper/StatisticsMapper.java | 1 - .../hotkey/dashboard/mapper/UserMapper.java | 4 +++- .../hotkey/dashboard/service/RuleService.java | 1 - .../service/impl/KeyServiceImpl.java | 1 - .../hotkey/dashboard/util/DateUtil.java | 1 - .../hotkey/dashboard/mapper/UserMapper.xml | 18 ++++++++++++++++- .../templates/admin/common/html/leftMenu.html | 2 +- .../templates/admin/common/html/topMenu.html | 20 +++++++++---------- 12 files changed, 37 insertions(+), 27 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java index 7bb37ef..2ea2d5f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/req/SearchReq.java @@ -6,7 +6,6 @@ import com.jd.platform.hotkey.dashboard.util.DateUtil; import java.io.Serializable; import java.time.LocalDateTime; import java.util.Date; -import java.util.List; /** * @ProjectName: hotkey diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index bc94342..0538405 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -4,7 +4,6 @@ package com.jd.platform.hotkey.dashboard.common.monitor; import com.ibm.etcd.api.Event; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; -import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.mapper.KeyRecordMapper; import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java index bd5a66a..5e8a52d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java @@ -4,9 +4,8 @@ import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.Page; -import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.Result; -import com.jd.platform.hotkey.dashboard.model.KeyRule; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.model.Rules; import com.jd.platform.hotkey.dashboard.service.RuleService; import org.springframework.stereotype.Controller; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java index cc7f6e5..80a48c7 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java @@ -1,17 +1,12 @@ package com.jd.platform.hotkey.dashboard.controller; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.Page; -import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.Result; +import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.model.User; import com.jd.platform.hotkey.dashboard.service.UserService; @@ -25,6 +20,10 @@ import javax.annotation.Resource; import javax.servlet.http.Cookie; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; @Controller diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java index fc5c79d..dabdfc5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/StatisticsMapper.java @@ -1,6 +1,5 @@ package com.jd.platform.hotkey.dashboard.mapper; -import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.model.Statistics; import org.apache.ibatis.annotations.Mapper; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java index 14d4b6c..cc6ca7a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/UserMapper.java @@ -10,7 +10,7 @@ import java.util.List; public interface UserMapper { int deleteByPrimaryKey(Integer id); - int insertSelective(User record); + int insertSelective(User user); User selectByPrimaryKey(Integer id); @@ -22,4 +22,6 @@ public interface UserMapper { List listApp(); + List selectHkUserList(User user); + } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java index ebd974e..f06e426 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java @@ -3,7 +3,6 @@ package com.jd.platform.hotkey.dashboard.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.model.KeyRule; import com.jd.platform.hotkey.dashboard.model.Rules; import java.util.List; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index d32e630..348e2bf 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -1,7 +1,6 @@ package com.jd.platform.hotkey.dashboard.service.impl; -import com.alibaba.fastjson.JSON; import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.ibm.etcd.api.Event; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java index bb49094..dbe252f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java @@ -2,7 +2,6 @@ package com.jd.platform.hotkey.dashboard.util; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.format.DateTimeFormatter; diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml index 87b2328..b1b238a 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/UserMapper.xml @@ -21,11 +21,27 @@ from hk_user where id = #{id,jdbcType=INTEGER} + delete from hk_user where id = #{id,jdbcType=INTEGER} - + insert into hk_user diff --git a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html index b1aea2e..3864a2d 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/leftMenu.html @@ -3,7 +3,7 @@
diff --git a/dashboard/src/main/resources/templates/admin/common/html/topMenu.html b/dashboard/src/main/resources/templates/admin/common/html/topMenu.html index 8715107..354cc91 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/topMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/topMenu.html @@ -89,24 +89,24 @@
    -
  •   Berlin  - 85 ; 15km/h +
  •   使命  +  -  科技引领生活
    • -
  •   Yogyakarta  - 85 ; Tonight- 72 °F (22.2 °C) +
  •   愿景  +  -  成为全球最值得信赖的企业
    • -
  •   Sttugart  - 85 ; 15km/h +
  •   核心价值观  +  -   客户为先 诚信 协作 感恩 拼搏 担当
    • -
  •   Muchen  +
@@ -123,7 +123,7 @@
- +
@@ -74,15 +74,15 @@ // 基于准备好的dom，初始化echarts实例 var myChart = echarts.init(document.getElementById('ruleLineChart')); - // 指定图表的配置项和数据 + // 指定图表的配置项和数据 var option = { xAxis: { type: 'category', data: ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] }, - // color:['#4472C5'], + // color:['#4472C5'], legend: { - data:['key21','key22','key23','key24','key25','key26','key27','key28','key29','key30','key31'], + data:['key21','key22'], itemWidth:80, itemHeight:25, itemGap:18, @@ -118,52 +118,6 @@ name: 'key22', data: [350, 255, 310, 494, 189, 333, 620], type: 'line' - }, - { - name: 'key23', - data: [310, 415, 350, 714, 219, 300, 500], - type: 'line' - }, - { - name: 'key24', - data: [380, 385, 370, 654, 569, 303, 620], - type: 'line' - }, - { - name: 'key25', - data: [320, 265, 300, 574, 389, 500, 700], - type: 'line' - }, - { - name: 'key26', - data: [390, 485, 320, 604, 429, 363, 620], - type: 'line' - }, - { - name: 'key27', - data: [300, 515, 580, 514, 259, 510, 430], - type: 'line' - }, - { - name: 'key28', - data: [280, 415, 380, 404, 429, 363, 480], - type: 'line' - }, - { - name: 'key29', - data: [450, 555, 320, 314, 559, 370, 460], - type: 'line' - } - , - { - name: 'key30', - data: [280, 615, 660, 444, 429, 363, 410], - type: 'line' - }, - { - name: 'key31', - data: [400, 555, 600, 333, 559, 310, 560], - type: 'line' } ] }; @@ -188,11 +142,11 @@ }, success : function(data) { console.log(data) - let selected_item = {}; + // let selected_item = {}; for (let i = 0; i < data.length; i++) { - selected_item[data[i]] = i === 0; + // selected_item[data[i]] = i === 0; option.legend.data.push(data[i]); - option.legend.selected = selected_item; + // option.legend.selected = selected_item; } } }); -- Gitee From e0a1c0688e541aa0a5bba256dcb79102d6e3cdf8 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 23 Jun 2020 12:41:22 +0800 Subject: [PATCH 22/51] =?UTF-8?q?=E7=BC=96=E8=BE=91=E5=AF=86=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/resources/static/admin/common/js/user-editPwd.js | 3 +++ .../src/main/resources/templates/admin/user/editPwd.html | 8 ++++---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js b/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js index 91ef833..eedeb56 100644 --- a/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js +++ b/dashboard/src/main/resources/static/admin/common/js/user-editPwd.js @@ -38,6 +38,9 @@ function edit() { url : "/user/editPwd", data : dataFormJson, async : false, + headers: { + "Authorization":getCookie("token") + }, error : function(request) { $.modal.alertError("系统错误"); }, diff --git a/dashboard/src/main/resources/templates/admin/user/editPwd.html b/dashboard/src/main/resources/templates/admin/user/editPwd.html index ccb9c3e..085cdf5 100644 --- a/dashboard/src/main/resources/templates/admin/user/editPwd.html +++ b/dashboard/src/main/resources/templates/admin/user/editPwd.html @@ -25,18 +25,18 @@
-
+
- +
- +
@@ -71,7 +71,7 @@ - + -- Gitee From fea5b2276db54e3197176e488fd2d4ad747ff224 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Mon, 29 Jun 2020 09:51:51 +0800 Subject: [PATCH 23/51] update README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 533c921..de1d687 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ ### 界面效果 ![输入图片说明](https://images.gitee.com/uploads/images/2020/0622/163805_0aa68d4b_303698.png "屏幕截图.png") ### 微信群 -![输入图片说明](https://images.gitee.com/uploads/images/2020/0622/100431_29c817c1_303698.jpeg "WechatIMG153.jpeg") +![输入图片说明](https://images.gitee.com/uploads/images/2020/0629/095135_4d6846df_303698.jpeg "WechatIMG157.jpeg") #### 安装教程 -- Gitee From 7c59f3619815086f9a9b9e0b5f9b1ecab61309b7 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 1 Jul 2020 12:07:07 +0800 Subject: [PATCH 24/51] merge --- .../platform/hotkey/client/ClientStarter.java | 1 + .../hotkey/client/callback/JdHotKeyStore.java | 48 ++++---- .../client/core/key/DefaultKeyHandler.java | 24 +++- .../hotkey/client/core/key/IKeyCollector.java | 14 ++- .../hotkey/client/core/key/IKeyHandler.java | 12 -- .../hotkey/client/core/key/IKeyPusher.java | 9 ++ .../client/core/key/KeyHandlerFactory.java | 22 ++-- .../hotkey/client/core/key/KeyHotModel.java | 32 +++++ .../client/core/key/NettyKeyPusher.java | 37 +++++- .../client/core/key/PushSchedulerStarter.java | 22 +++- .../client/core/key/TurnCountCollector.java | 115 ++++++++++++++++++ .../client/core/key/TurnKeyCollector.java | 19 ++- .../client/core/rule/KeyRuleHolder.java | 15 ++- .../hotkey/common/model/HotKeyMsg.java | 15 +++ .../hotkey/common/model/KeyCountModel.java | 67 ++++++++++ .../common/model/typeenum/MessageType.java | 1 + .../platform/hotkey/common/tool/Constant.java | 7 ++ .../platform/hotkey/common/tool/TwoTuple.java | 23 ++++ 18 files changed, 411 insertions(+), 72 deletions(-) delete mode 100644 client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyHandler.java create mode 100644 client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHotModel.java create mode 100644 client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java create mode 100644 common/src/main/java/com/jd/platform/hotkey/common/model/KeyCountModel.java create mode 100644 common/src/main/java/com/jd/platform/hotkey/common/tool/TwoTuple.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java index f6267a3..1102d8a 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/ClientStarter.java @@ -98,6 +98,7 @@ public class ClientStarter { EtcdConfigFactory.buildConfigCenter(etcdServer); //开始定时推送 PushSchedulerStarter.startPusher(pushPeriod); + PushSchedulerStarter.startCountPusher(10); //开启worker重连器 WorkerRetryConnector.retryConnectWorkers(); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java index 9bf7d35..ee43413 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java @@ -3,6 +3,8 @@ package com.jd.platform.hotkey.client.callback; import com.jd.platform.hotkey.client.cache.CacheFactory; import com.jd.platform.hotkey.client.cache.LocalCache; import com.jd.platform.hotkey.client.core.key.HotKeyPusher; +import com.jd.platform.hotkey.client.core.key.KeyHandlerFactory; +import com.jd.platform.hotkey.client.core.key.KeyHotModel; import com.jd.platform.hotkey.common.model.typeenum.KeyType; import com.jd.platform.hotkey.common.tool.Constant; @@ -20,11 +22,13 @@ public class JdHotKeyStore { if (!inRule(key)) { return false; } - Object value = getValueSimple(key); - if (value == null) { + boolean isHot = isHot(key); + if (!isHot) { HotKeyPusher.push(key, null); } - return value != null; + //统计计数 + KeyHandlerFactory.getCounter().collect(new KeyHotModel(key, isHot)); + return isHot; } /** @@ -43,7 +47,9 @@ public class JdHotKeyStore { * 判断是否是热key，如果是热key，则给value赋值 */ public static void smartSet(String key, Object value) { - setValue(key, value); + if (isHot(key)) { + setValueDirectly(key, value); + } } /** @@ -58,6 +64,8 @@ public class JdHotKeyStore { if (value == null) { HotKeyPusher.push(key, keyType); } + //统计计数 + KeyHandlerFactory.getCounter().collect(new KeyHotModel(key, value != null)); //如果是默认值，也返回null if(value instanceof Integer && Constant.MAGIC_NUMBER == (int) value) { return null; @@ -76,16 +84,10 @@ public class JdHotKeyStore { return getCache(key).get(key); } - public static void setValue(String key, Object value) { - if (isHot(key)) { - setValueDirectly(key, value); - } - } - /** * 纯粹的本地缓存，无需该key是热key */ - public static void setValueDirectly(String key, Object value) { + static void setValueDirectly(String key, Object value) { getCache(key).set(key, value); } @@ -98,28 +100,28 @@ public class JdHotKeyStore { /** * 判断是否是热key。适用于只需要判断key，而不需要value的场景 */ - public static boolean isHot(String key) { + static boolean isHot(String key) { return getValueSimple(key) != null; } /** * 判断是否已经缓存过了。 */ - public static boolean isValueCached(String key, KeyType keyType) { - Object value = getValue(key, keyType); - //如果value不为null且不为默认值 - if (value == null) { - return false; - } - return !(value instanceof Integer) || Constant.MAGIC_NUMBER != (int) value; - } +// private static boolean isValueCached(String key, KeyType keyType) { +// Object value = getValue(key, keyType); +// //如果value不为null且不为默认值 +// if (value == null) { +// return false; +// } +// return !(value instanceof Integer) || Constant.MAGIC_NUMBER != (int) value; +// } /** * 判断某key的value是否已经缓存过了 */ - public static boolean isValueCached(String key) { - return isValueCached(key, KeyType.REDIS_KEY); - } +// public static boolean isValueCached(String key) { +// return isValueCached(key, KeyType.REDIS_KEY); +// } private static LocalCache getCache(String key) { return CacheFactory.getNonNullCache(key); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/DefaultKeyHandler.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/DefaultKeyHandler.java index bfc0a8d..33b71f3 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/DefaultKeyHandler.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/DefaultKeyHandler.java @@ -1,17 +1,29 @@ package com.jd.platform.hotkey.client.core.key; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.KeyCountModel; + /** * @author wuweifeng wrote on 2020-02-25 * @version 1.0 */ -public class DefaultKeyHandler implements IKeyHandler { - @Override +public class DefaultKeyHandler { + private IKeyPusher iKeyPusher = new NettyKeyPusher(); + + private IKeyCollector iKeyCollector = new TurnKeyCollector(); + + private IKeyCollector iKeyCounter = new TurnCountCollector(); + + public IKeyPusher keyPusher() { - return new NettyKeyPusher(); + return iKeyPusher; + } + + public IKeyCollector keyCollector() { + return iKeyCollector; } - @Override - public IKeyCollector keyCollector() { - return new TurnKeyCollector(); + public IKeyCollector keyCounter() { + return iKeyCounter; } } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyCollector.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyCollector.java index e53de7f..c2a5fa2 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyCollector.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyCollector.java @@ -1,7 +1,5 @@ package com.jd.platform.hotkey.client.core.key; -import com.jd.platform.hotkey.common.model.HotKeyModel; - import java.util.List; /** @@ -9,10 +7,16 @@ import java.util.List; * @author wuweifeng wrote on 2020-01-06 * @version 1.0 */ -public interface IKeyCollector { - List lockAndGetResult(); +public interface IKeyCollector { + /** + * 锁定后的返回值 + */ + List lockAndGetResult(); - void collect(HotKeyModel hotKeyModel); + /** + * 输入的参数 + */ + void collect(T t); void finishOnce(); } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyHandler.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyHandler.java deleted file mode 100644 index 64b33c2..0000000 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyHandler.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.jd.platform.hotkey.client.core.key; - -/** - * @author wuweifeng wrote on 2020-02-25 - * @version 1.0 - */ -public interface IKeyHandler { - - IKeyPusher keyPusher(); - - IKeyCollector keyCollector(); -} diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyPusher.java index 7d70b57..c8c706f 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/IKeyPusher.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.client.core.key; import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.KeyCountModel; import java.util.List; @@ -10,5 +11,13 @@ import java.util.List; * @version 1.0 */ public interface IKeyPusher { + /** + * 发送待测key + */ void send(String appName, List list); + + /** + * 发送热key访问量 + */ + void sendCount(String appName, List list); } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHandlerFactory.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHandlerFactory.java index 0453446..f9fd24d 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHandlerFactory.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHandlerFactory.java @@ -1,25 +1,27 @@ package com.jd.platform.hotkey.client.core.key; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.KeyCountModel; + /** * @author wuweifeng wrote on 2020-01-06 * @version 1.0 */ public class KeyHandlerFactory { - private static final IKeyHandler iKeyHandler = new DefaultKeyHandler(); - - private static final IKeyPusher iKeyPusher = iKeyHandler.keyPusher(); - - private static final IKeyCollector iKeyCollector = iKeyHandler.keyCollector(); - + private static final DefaultKeyHandler iKeyHandler = new DefaultKeyHandler(); private KeyHandlerFactory() { } - public synchronized static IKeyPusher getPusher() { - return iKeyPusher; + public static IKeyPusher getPusher() { + return iKeyHandler.keyPusher(); + } + + public static IKeyCollector getCollector() { + return iKeyHandler.keyCollector(); } - public synchronized static IKeyCollector getCollector() { - return iKeyCollector; + public static IKeyCollector getCounter() { + return iKeyHandler.keyCounter(); } } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHotModel.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHotModel.java new file mode 100644 index 0000000..96df161 --- /dev/null +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/KeyHotModel.java @@ -0,0 +1,32 @@ +package com.jd.platform.hotkey.client.core.key; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-06-24 + */ +public class KeyHotModel { + private String key; + private boolean isHot; + + public KeyHotModel(String key, boolean isHot) { + this.key = key; + this.isHot = isHot; + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public boolean isHot() { + return isHot; + } + + public void setHot(boolean hot) { + isHot = hot; + } +} diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java index fca5cf2..983b5a8 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java @@ -4,6 +4,7 @@ import com.jd.platform.hotkey.client.core.worker.WorkerInfoHolder; import com.jd.platform.hotkey.client.log.JdLogger; import com.jd.platform.hotkey.common.model.HotKeyModel; import com.jd.platform.hotkey.common.model.HotKeyMsg; +import com.jd.platform.hotkey.common.model.KeyCountModel; import com.jd.platform.hotkey.common.model.MsgBuilder; import com.jd.platform.hotkey.common.model.typeenum.MessageType; import com.jd.platform.hotkey.common.tool.FastJsonUtils; @@ -35,8 +36,8 @@ public class NettyKeyPusher implements IKeyPusher { continue; } - map.computeIfAbsent(channel, k -> new ArrayList<>()); - map.get(channel).add(model); + List newList = map.computeIfAbsent(channel, k -> new ArrayList<>()); + newList.add(model); } for (Channel channel : map.keySet()) { @@ -56,4 +57,36 @@ public class NettyKeyPusher implements IKeyPusher { } + @Override + public void sendCount(String appName, List list) { + //积攒了10秒的数量，按照hash分发到不同的worker + long now = System.currentTimeMillis(); + Map> map = new HashMap<>(); + for(KeyCountModel model : list) { + model.setCreateTime(now); + Channel channel = WorkerInfoHolder.chooseChannel(model.getRuleKey()); + if (channel == null) { + continue; + } + + List newList = map.computeIfAbsent(channel, k -> new ArrayList<>()); + newList.add(model); + } + + for (Channel channel : map.keySet()) { + try { + List batch = map.get(channel); + channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_HIT_COUNT, FastJsonUtils.convertObjectToJSON(batch)))); + } catch (Exception e) { + try { + InetSocketAddress insocket = (InetSocketAddress) channel.remoteAddress(); + JdLogger.error(getClass(),"flush error " + insocket.getAddress().getHostAddress()); + } catch (Exception ex) { + JdLogger.error(getClass(),"flush error"); + } + + } + } + } + } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/PushSchedulerStarter.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/PushSchedulerStarter.java index a77439e..b5eda05 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/PushSchedulerStarter.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/PushSchedulerStarter.java @@ -1,6 +1,8 @@ package com.jd.platform.hotkey.client.core.key; import com.jd.platform.hotkey.client.Context; +import com.jd.platform.hotkey.common.model.HotKeyModel; +import com.jd.platform.hotkey.common.model.KeyCountModel; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -13,16 +15,34 @@ import java.util.concurrent.TimeUnit; */ public class PushSchedulerStarter { + /** + * 每0.5秒推送一次待测key + */ public static void startPusher(Long period) { if (period == null || period <= 0) { period = 500L; } ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); scheduledExecutorService.scheduleAtFixedRate(() -> { - IKeyCollector collectHK = KeyHandlerFactory.getCollector(); + IKeyCollector collectHK = KeyHandlerFactory.getCollector(); KeyHandlerFactory.getPusher().send(Context.APP_NAME, collectHK.lockAndGetResult()); collectHK.finishOnce(); },0, period, TimeUnit.MILLISECONDS); } + /** + * 每10秒推送一次数量统计 + */ + public static void startCountPusher(Integer period) { + if (period == null || period <= 0) { + period = 10; + } + ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); + scheduledExecutorService.scheduleAtFixedRate(() -> { + IKeyCollector collectHK = KeyHandlerFactory.getCounter(); + KeyHandlerFactory.getPusher().sendCount(Context.APP_NAME, collectHK.lockAndGetResult()); + collectHK.finishOnce(); + },0, period, TimeUnit.SECONDS); + } + } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java new file mode 100644 index 0000000..329ec7c --- /dev/null +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnCountCollector.java @@ -0,0 +1,115 @@ +package com.jd.platform.hotkey.client.core.key; + +import cn.hutool.core.util.StrUtil; +import com.jd.platform.hotkey.client.core.rule.KeyRuleHolder; +import com.jd.platform.hotkey.common.model.KeyCountModel; +import com.jd.platform.hotkey.common.tool.Constant; + +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 热点数量统计 + * @author wuweifeng + * @version 1.0 + * @date 2020-06-24 + */ +public class TurnCountCollector implements IKeyCollector { + /** + * 存储格式为： + * pin_20200624091024 -> 10 + * pin_20200624091025 -> 20 + *

+ * sku_20200624091024 -> 123 + * sku_20200624091025 -> 142 + */ + private ConcurrentHashMap HIT_MAP_0 = new ConcurrentHashMap<>(512); + private ConcurrentHashMap HIT_MAP_1 = new ConcurrentHashMap<>(512); + + private static final String FORMAT = "yyyy-MM-dd HH:mm:ss"; + + private AtomicLong atomicLong = new AtomicLong(0); + + @Override + public List lockAndGetResult() { + //自增后，对应的map就会停止被写入，等待被读取 + atomicLong.addAndGet(1); + + List list; + if (atomicLong.get() % 2 == 0) { + list = get(HIT_MAP_1); + HIT_MAP_1.clear(); + } else { + list = get(HIT_MAP_0); + HIT_MAP_0.clear(); + } + return list; + } + + /** + * 每10秒上报一次最近10秒的数据，并且删掉相应的key + */ + private List get(ConcurrentHashMap map) { + List list = new ArrayList<>(); + for (Map.Entry entry : map.entrySet()) { + String key = entry.getKey(); + HitCount hitCount = entry.getValue(); + KeyCountModel keyCountModel = new KeyCountModel(); + keyCountModel.setTotalHitCount(hitCount.totalHitCount.get()); + keyCountModel.setRuleKey(key); + keyCountModel.setHotHitCount(hitCount.hotHitCount.get()); + list.add(keyCountModel); + } + return list; + } + + @Override + public void collect(KeyHotModel keyHotModel) { + if (atomicLong.get() % 2 == 0) { + put(keyHotModel.getKey(), keyHotModel.isHot(), HIT_MAP_0); + } else { + put(keyHotModel.getKey(), keyHotModel.isHot(), HIT_MAP_1); + } + } + + @Override + public void finishOnce() { + + } + + public void put(String key, boolean isHot, ConcurrentHashMap map) { + //如key是pin_的前缀，则存储pin_ + String rule = KeyRuleHolder.rule(key); + //不在规则内的不处理 + if (StrUtil.isEmpty(rule)) { + return; + } + String nowTime = nowTime(); + + //rule + 分隔符 + 2020-10-23 21:11:22 + String mapKey = rule + Constant.COUNT_DELIMITER + nowTime; + //该方法线程安全 + HitCount hitCount = map.computeIfAbsent(mapKey, v -> new HitCount()); + if (isHot) { + hitCount.hotHitCount.incrementAndGet(); + } + hitCount.totalHitCount.incrementAndGet(); + } + + private String nowTime() { + Date nowTime = new Date(System.currentTimeMillis()); + SimpleDateFormat sdFormatter = new SimpleDateFormat(FORMAT); + return sdFormatter.format(nowTime); + } + + private class HitCount { + private AtomicInteger hotHitCount = new AtomicInteger(); + private AtomicInteger totalHitCount = new AtomicInteger(); + } +} diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnKeyCollector.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnKeyCollector.java index 87e9031..353938c 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnKeyCollector.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/TurnKeyCollector.java @@ -16,7 +16,7 @@ import java.util.concurrent.atomic.AtomicLong; * @author wuweifeng wrote on 2020-01-06 * @version 1.0 */ -public class TurnKeyCollector implements IKeyCollector { +public class TurnKeyCollector implements IKeyCollector { private ConcurrentHashMap map0 = new ConcurrentHashMap<>(); private ConcurrentHashMap map1 = new ConcurrentHashMap<>(); @@ -49,21 +49,15 @@ public class TurnKeyCollector implements IKeyCollector { return; } if (atomicLong.get() % 2 == 0) { - HotKeyModel model = map0.get(key); - if (model == null) { - map0.put(key, hotKeyModel); - } else { + //不存在时返回null并将key-value放入，已有相同key时，返回该key对应的value，并且不覆盖 + HotKeyModel model = map0.putIfAbsent(key, hotKeyModel); + if (model != null) { model.setCount(model.getCount() + hotKeyModel.getCount()); - map0.put(key, model); } - } else { - HotKeyModel model = map1.get(key); - if (model == null) { - map1.put(key, hotKeyModel); - } else { + HotKeyModel model = map1.putIfAbsent(key, hotKeyModel); + if (model != null) { model.setCount(model.getCount() + hotKeyModel.getCount()); - map1.put(key, model); } } @@ -73,4 +67,5 @@ public class TurnKeyCollector implements IKeyCollector { public void finishOnce() { } + } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java b/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java index 79dd504..4397d63 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java @@ -101,6 +101,19 @@ public class KeyRuleHolder { } + /** + * 判断该key命中了哪个rule + */ + public static String rule(String key) { + String rule = ""; + for (KeyRule keyRule : KEY_RULES) { + if (key.equals(keyRule.getKey()) || (keyRule.isPrefix() && key.startsWith(keyRule.getKey())) || "*".equals(keyRule.getKey())) { + return keyRule.getKey(); + } + } + return rule; + } + /** * 判断key是否在配置的要探测的规则内 */ @@ -121,7 +134,7 @@ public class KeyRuleHolder { @Subscribe public void ruleChange(KeyRuleInfoChangeEvent event) { - JdLogger.info(getClass(),"new rules info is :" + event.getKeyRules()); + JdLogger.info(getClass(), "new rules info is :" + event.getKeyRules()); List ruleList = event.getKeyRules(); if (ruleList == null) { return; diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java index 4e197f3..a1caf1a 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/HotKeyMsg.java @@ -10,11 +10,18 @@ import com.jd.platform.hotkey.common.model.typeenum.MessageType; public class HotKeyMsg { private int magicNumber; + private String appName; + private MessageType messageType; private String body; public HotKeyMsg(MessageType messageType, String body) { + this(null, messageType, body); + } + + public HotKeyMsg(String appName, MessageType messageType, String body) { + this.appName = appName; this.messageType = messageType; this.body = body; } @@ -31,6 +38,14 @@ public class HotKeyMsg { '}'; } + public String getAppName() { + return appName; + } + + public void setAppName(String appName) { + this.appName = appName; + } + public int getMagicNumber() { return magicNumber; } diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/KeyCountModel.java b/common/src/main/java/com/jd/platform/hotkey/common/model/KeyCountModel.java new file mode 100644 index 0000000..09f3563 --- /dev/null +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/KeyCountModel.java @@ -0,0 +1,67 @@ +package com.jd.platform.hotkey.common.model; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-06-24 + */ +public class KeyCountModel { + /** + * 对应的规则名，如 pin_2020-08-09 11:32:43 + */ + private String ruleKey; + /** + * 总访问次数 + */ + private int totalHitCount; + /** + * 热后访问次数 + */ + private int hotHitCount; + /** + * 发送时的时间 + */ + private long createTime; + + @Override + public String toString() { + return "KeyCountModel{" + + "ruleKey='" + ruleKey + '\'' + + ", totalHitCount=" + totalHitCount + + ", hotHitCount=" + hotHitCount + + ", createTime=" + createTime + + '}'; + } + + public long getCreateTime() { + return createTime; + } + + public void setCreateTime(long createTime) { + this.createTime = createTime; + } + + public String getRuleKey() { + return ruleKey; + } + + public void setRuleKey(String ruleKey) { + this.ruleKey = ruleKey; + } + + public int getTotalHitCount() { + return totalHitCount; + } + + public void setTotalHitCount(int totalHitCount) { + this.totalHitCount = totalHitCount; + } + + public int getHotHitCount() { + return hotHitCount; + } + + public void setHotHitCount(int hotHitCount) { + this.hotHitCount = hotHitCount; + } +} diff --git a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java index 57daa2b..cd3f983 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/model/typeenum/MessageType.java @@ -8,6 +8,7 @@ public enum MessageType { APP_NAME((byte) 1), REQUEST_NEW_KEY((byte) 2), RESPONSE_NEW_KEY((byte) 3), + REQUEST_HIT_COUNT((byte) 7), PING((byte) 4), PONG((byte) 5), EMPTY((byte) 6); diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java index 9e0cc85..0c9bf86 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java @@ -14,7 +14,14 @@ public class Constant { public static int Default_Threads = 4; public static int MAGIC_NUMBER = 0x12fcf76; + /** + * netty的分隔符 + */ public static String DELIMITER = "$(* *)$"; + /** + * 数量统计时，rule+时间 组成key用的分隔符 + */ + public static String COUNT_DELIMITER = "#**#"; //单次包最大2M public static int MAX_LENGTH = 2 * 1024 * 1024; diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/TwoTuple.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/TwoTuple.java new file mode 100644 index 0000000..9aa4051 --- /dev/null +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/TwoTuple.java @@ -0,0 +1,23 @@ +package com.jd.platform.hotkey.common.tool; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-06-24 + */ +public class TwoTuple { + + public final A first; + + public final B second; + + public TwoTuple(A a, B b){ + first = a; + second = b; + } + + public String toString(){ + return "(" + first + ", " + second + ")"; + } + +} \ No newline at end of file -- Gitee From 1f4aad780e8911c8d97a733244e79b6b439b3dd5 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Thu, 2 Jul 2020 12:01:41 +0800 Subject: [PATCH 25/51] update README.md. --- README.md | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index de1d687..10ca779 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,10 @@ 该框架历经多次压测，8核单机worker端每秒可接收处理16万个key探测任务，16核单机至少每秒平稳处理20万以上，实际压测达到30万以上，CPU平稳支撑，框架无异常。在真实业务场景中，可用1：1000的比例，即1台worker支撑1000台服务端Tomcat的key探测任务，即可带来极大的数据存储资源节省（如对redis集群的扩充）。测试详情可去我[CSDN博客](https://blog.csdn.net/tianyaleixiaowu)查看。 +#### 架构设计 +见京东零售公众号 +https://mp.weixin.qq.com/s/xOzEj5HtCeh_ezHDPHw6Jw + #### 介绍 对任意突发性的无法预先感知的热点请求，包括并不限于热点数据（如突发大量请求同一个商品）、热用户（如爬虫、刷子）、热接口（突发海量请求同一个接口）等，进行毫秒级精准探测到。 然后对这些热数据、热用户等，推送到该应用部署的所有机器JVM内存中，以大幅减轻对后端数据存储层的冲击，并可以由客户端决定如何使用这些热key（譬如对热商品做本地缓存、对热用户进行拒绝访问、对热接口进行熔断或返回默认值）。 @@ -160,20 +164,3 @@ Object JdHotKeyStore.getValue(String key) - -#### 参与贡献 - -1. Fork 本仓库 -2. 新建 Feat_xxx 分支 -3. 提交代码 -4. 新建 Pull Request - - -#### 码云特技 - -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md -2. 码云官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解码云上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是码云最有价值开源项目，是码云综合评定出的优秀开源项目 -5. 码云官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. 码云封面人物是一档用来展示码云会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) -- Gitee From 33c865b548b3437647a8d4ff1ae9f3196ebb1bb2 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Thu, 2 Jul 2020 12:13:13 +0800 Subject: [PATCH 26/51] update README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 10ca779..2a6c5d9 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ public void initHotkey() { starter.startPipeline(); } ``` -其中还可以setCaffeineSize(int size)设置本地缓存最大数量，默认5万，setPushPeriod(Long period)设置批量推送key的间隔时间，默认500ms，该值越小，上报热key越频繁，相应越准确，但消耗资源越多。 +其中还可以setCaffeineSize(int size)设置本地缓存最大数量，默认5万，setPushPeriod(Long period)设置批量推送key的间隔时间，默认500ms，该值越小，上报热key越频繁，相应越及时，建议根据实际情况调整，如单机每秒qps10个，那么0.5秒上报一次即可，否则是空跑。该值最小为1，即1ms上报一次。 主要有如下4个方法可供使用 -- Gitee From e63789c093d659bbc0418c0bec7f1dd1b6aad385 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Fri, 3 Jul 2020 11:22:20 +0800 Subject: [PATCH 27/51] update README.md. --- README.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a6c5d9..bea21ce 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,20 @@ 该框架历经多次压测，8核单机worker端每秒可接收处理16万个key探测任务，16核单机至少每秒平稳处理20万以上，实际压测达到30万以上，CPU平稳支撑，框架无异常。在真实业务场景中，可用1：1000的比例，即1台worker支撑1000台服务端Tomcat的key探测任务，即可带来极大的数据存储资源节省（如对redis集群的扩充）。测试详情可去我[CSDN博客](https://blog.csdn.net/tianyaleixiaowu)查看。 #### 架构设计 -见京东零售公众号 + **见京东零售公众号** https://mp.weixin.qq.com/s/xOzEj5HtCeh_ezHDPHw6Jw +#### 参与开发 +该框架的开发得到了来自于： + +京东数科-生活服务研发组-李云峰的大力支持，完成了整个dashboard控制台的研发。 + +京东零售-平台业务中心-APP后台架构师 杜晓宇的大力支持，参与并帮助解决了诸多技术难题。 + +另有京东集团多个部门提供的意见、建议，和对框架的见解、优化方案。 + +在此一起感谢。 + #### 介绍 对任意突发性的无法预先感知的热点请求，包括并不限于热点数据（如突发大量请求同一个商品）、热用户（如爬虫、刷子）、热接口（突发海量请求同一个接口）等，进行毫秒级精准探测到。 然后对这些热数据、热用户等，推送到该应用部署的所有机器JVM内存中，以大幅减轻对后端数据存储层的冲击，并可以由客户端决定如何使用这些热key（譬如对热商品做本地缓存、对热用户进行拒绝访问、对热接口进行熔断或返回默认值）。 -- Gitee From e76d0e4828feeceb1ba503e5f2b6f699c6d66f79 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Sun, 5 Jul 2020 10:20:34 +0800 Subject: [PATCH 28/51] =?UTF-8?q?=E5=88=A0=E9=99=A4erp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../interceptor/MyWebAppConfigurer.java | 8 +--- .../hotkey/dashboard/service/UserService.java | 4 -- .../service/impl/RuleServiceImpl.java | 2 +- .../service/impl/UserServiceImpl.java | 42 ------------------- 4 files changed, 3 insertions(+), 53 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java index dc8306e..135b277 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/MyWebAppConfigurer.java @@ -1,22 +1,18 @@ package com.jd.platform.hotkey.dashboard.interceptor; -import com.jd.platform.hotkey.dashboard.erp.ErpUimInterceptor; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Configuration; import org.springframework.core.Ordered; import org.springframework.web.servlet.config.annotation.*; + @Configuration public class MyWebAppConfigurer implements WebMvcConfigurer { - @Autowired - ErpUimInterceptor erpUimInterceptor; @Override public void addInterceptors(InterceptorRegistry registry) { //拦截路径可自行配置多个 可用 ，分隔开 InterceptorRegistration registration = registry.addInterceptor(new JwtInterceptor()).addPathPatterns("/**"); - registry.addInterceptor(erpUimInterceptor); - registration.excludePathPatterns("/user/login","/user/getUserName","/error","/static/**","/main/**"); + registration.excludePathPatterns("/user/login","/error","/static/**","/main/**"); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java index ae01777..08f294a 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/UserService.java @@ -23,10 +23,6 @@ public interface UserService { int insertUser(User user); - int insertUserByErp(User user); - - Cookie loginErpUser(User user); - int deleteByPrimaryKey(int id); User selectByPrimaryKey(int id); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java index 2a85103..b1d2c04 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java @@ -15,9 +15,9 @@ import com.jd.platform.hotkey.dashboard.model.ChangeLog; import com.jd.platform.hotkey.dashboard.model.Rule; import com.jd.platform.hotkey.dashboard.model.Rules; import com.jd.platform.hotkey.dashboard.service.RuleService; -import org.apache.commons.lang3.StringUtils; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; +import org.springframework.util.StringUtils; import javax.annotation.Resource; import java.util.ArrayList; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java index dab819c..3bfab9c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java @@ -34,8 +34,6 @@ import java.util.Set; @Service public class UserServiceImpl implements UserService { - @Value("${erp.defaultPwd}") - private String defaultPwd; @Resource private UserMapper userMapper; @Resource @@ -68,46 +66,6 @@ public class UserServiceImpl implements UserService { return userMapper.insertSelective(user); } - @Override - public int insertUserByErp(User user) { - User userParam = new User(); - userParam.setUserName(user.getUserName()); - List users = userMapper.selectHkUserList(userParam); - if(users.size() == 0){ - user.setCreateTime(new Date()); - user.setPwd(DigestUtils.md5DigestAsHex(defaultPwd.getBytes())); - int ret = userMapper.insertSelective(user); - System.out.println(user.getId()); - return ret; - } - return 0; - } - - @Override - public Cookie loginErpUser(User user){ - User userParam = new User(); - userParam.setUserName(user.getUserName()); - List users = userMapper.selectHkUserList(userParam); - if(users.size() == 0){ - user.setCreateTime(new Date()); - user.setPwd(DigestUtils.md5DigestAsHex(defaultPwd.getBytes())); - userMapper.insertSelective(user); - String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), "", user.getAppName(), user.getNickName()); - Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); - cookie.setMaxAge(3600*24*7); - //cookie.setDomain("localhost"); - cookie.setPath("/"); - return cookie; - }else{ - user =users.get(0); - String token = JwtTokenUtil.createJWT(user.getId(), user.getUserName(), "", user.getAppName(), user.getNickName()); - Cookie cookie = new Cookie("token", JwtTokenUtil.TOKEN_PREFIX + token); - cookie.setMaxAge(3600*24*7); - //cookie.setDomain("localhost"); - cookie.setPath("/"); - return cookie; - } - } @Override public int deleteByPrimaryKey(int id) { -- Gitee From b5523e6d00d467cc501e4210de62e84992f27acc Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Mon, 6 Jul 2020 09:57:23 +0800 Subject: [PATCH 29/51] update README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bea21ce..a4a60f2 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ https://mp.weixin.qq.com/s/xOzEj5HtCeh_ezHDPHw6Jw ### 界面效果 ![输入图片说明](https://images.gitee.com/uploads/images/2020/0622/163805_0aa68d4b_303698.png "屏幕截图.png") ### 微信群 -![输入图片说明](https://images.gitee.com/uploads/images/2020/0629/095135_4d6846df_303698.jpeg "WechatIMG157.jpeg") +![输入图片说明](https://images.gitee.com/uploads/images/2020/0706/095712_fc4d8d97_303698.jpeg "11.jpeg") #### 安装教程 -- Gitee From 408a54a53f0d8ce8fec3af1c1b26ecbbfcfd6dae Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Mon, 6 Jul 2020 19:12:44 +0800 Subject: [PATCH 30/51] =?UTF-8?q?worker=20caffeinesize=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../hotkey/worker/cache/CaffeineBuilder.java | 3 +- .../hotkey/worker/mapper/AppInfoMapper.java | 20 ------- .../hotkey/worker/mapper/AppInfoMapper.xml | 55 ------------------- .../hotkey/worker/starters/InitStarter.java | 16 ++++-- .../hotkey/worker/tool/InitConstant.java | 5 ++ 5 files changed, 17 insertions(+), 82 deletions(-) delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/mapper/AppInfoMapper.java delete mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/mapper/AppInfoMapper.xml diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java index ad8b4e9..312560a 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/cache/CaffeineBuilder.java @@ -2,6 +2,7 @@ package com.jd.platform.hotkey.worker.cache; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import com.jd.platform.hotkey.worker.tool.InitConstant; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -21,7 +22,7 @@ public class CaffeineBuilder { return Caffeine.newBuilder() .initialCapacity(8192)//初始大小 .maximumSize(5000000)//最大数量。这个数值我设置的很大，按30万每秒，每分钟是1800万，实际可以调小 - .expireAfterWrite(1, TimeUnit.MINUTES)//过期时间 + .expireAfterWrite(InitConstant.caffeineMaxMinutes, TimeUnit.MINUTES)//过期时间，默认1分钟 .executor(executorService) .softValues() .build(); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/mapper/AppInfoMapper.java b/worker/src/main/java/com/jd/platform/hotkey/worker/mapper/AppInfoMapper.java deleted file mode 100644 index 3f50fab..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/mapper/AppInfoMapper.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.jd.platform.hotkey.worker.mapper; - -import com.jd.platform.hotkey.worker.model.AppInfo; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Options; - -/** - * AppInfoMapper - * - * @author longhuashen - * @since 2020-05-18 - */ -@Mapper -public interface AppInfoMapper { - - @Options(useGeneratedKeys = true) - int insertSelective(AppInfo appInfo); - - AppInfo findByAppName(String appName); -} \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/mapper/AppInfoMapper.xml b/worker/src/main/java/com/jd/platform/hotkey/worker/mapper/AppInfoMapper.xml deleted file mode 100644 index cb6cae6..0000000 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/mapper/AppInfoMapper.xml +++ /dev/null @@ -1,55 +0,0 @@ - - - - - - insert ignore into hk_app_info - - - id, - - - app_name, - - - principal_name, - - - principal_phone, - - - app_desc, - - - create_time, - - - - - #{id,jdbcType=INTEGER}, - - - #{appName,jdbcType=VARCHAR}, - - - #{principalName,jdbcType=VARCHAR}, - - - #{principalPhone,jdbcType=VARCHAR}, - - - #{appDesc,jdbcType=VARCHAR}, - - - #{createTime,jdbcType=TIMESTAMP}, - - - - - - \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/InitStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/InitStarter.java index 35645b2..42f8fa1 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/InitStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/InitStarter.java @@ -2,26 +2,30 @@ package com.jd.platform.hotkey.worker.starters; import com.jd.platform.hotkey.worker.tool.InitConstant; import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -import javax.annotation.PostConstruct; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; /** * @author wuweifeng * @version 1.0 * @date 2020-05-22 */ -@Component +@Configuration public class InitStarter { @Value("${netty.timeOut}") private int timeOut; @Value("${disruptor.bufferSize}") private int bufferSize; + @Value("${caffeine.minutes}") + private int caffeineMinutes; - @PostConstruct - public void init() { + @Bean("initParam") + public Object init() { InitConstant.timeOut = timeOut; InitConstant.bufferSize = bufferSize; + InitConstant.caffeineMaxMinutes = caffeineMinutes; + + return null; } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java index 4595218..f9702f2 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/InitConstant.java @@ -22,4 +22,9 @@ public class InitConstant { public static final LongAdder totalDealCount = new LongAdder(); public static final LongAdder totalOfferCount = new LongAdder(); + + /** + * key在caffeine里多久过期，默认只存1分钟的 + */ + public static int caffeineMaxMinutes = 1; } -- Gitee From e201a3583ff1f6a3373044767f3dc78790bdf4b0 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Mon, 6 Jul 2020 19:18:57 +0800 Subject: [PATCH 31/51] =?UTF-8?q?=E5=88=A4=E6=96=ADJava=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E8=8E=B7=E5=8F=96docker=E5=86=85cpu=E6=A0=B8=E6=95=B0=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- worker/src/main/resources/application.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/worker/src/main/resources/application.yml b/worker/src/main/resources/application.yml index eafa7ca..80b4968 100644 --- a/worker/src/main/resources/application.yml +++ b/worker/src/main/resources/application.yml @@ -6,6 +6,8 @@ open: timeout: ${openTimeOut:true} thread: count: ${threadCount:0} +caffeine: + minutes: ${caffeineMinutes:1} disruptor: bufferSize: ${bufferSize:2} #必须是2的整数倍 #etcd的地址，如有多个用逗号分隔 -- Gitee From c187a886ab0ff65c573242a84ebe67d530b66082 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 7 Jul 2020 19:22:02 +0800 Subject: [PATCH 32/51] =?UTF-8?q?sql=E8=A1=A5=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dashboard/src/main/resources/db.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index fb739f2..cc7a600 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -65,6 +65,7 @@ CREATE TABLE `hk_key_record` ( `type` int(11) NOT NULL DEFAULT 1 COMMENT '记录类型：1put；2del; -1unkonw', `create_time` datetime(0) NOT NULL ON UPDATE CURRENT_TIMESTAMP(0) COMMENT '创建时间', `uuid` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '防重ID', + `rule` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '''' COMMENT '规则', PRIMARY KEY (`id`) USING BTREE, UNIQUE INDEX `uniq_key`(`uuid`) USING BTREE COMMENT '唯一索引' ) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; -- Gitee From 6dd6e856a880545d0dea6f2780362151716bb051 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 8 Jul 2020 16:17:36 +0800 Subject: [PATCH 33/51] =?UTF-8?q?counter=E5=8A=9F=E8=83=BD=E5=8A=A0?= =?UTF-8?q?=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/callback/AbsReceiveNewKey.java | 14 ++- .../callback/DefaultNewKeyListener.java | 8 +- .../hotkey/client/callback/JdHotKeyStore.java | 86 ++++++++++++------- .../hotkey/client/callback/ValueModel.java | 62 +++++++++++++ .../client/core/key/NettyKeyPusher.java | 4 +- .../client/core/rule/KeyRuleHolder.java | 65 ++++++++------ .../common/configcenter/ConfigConstant.java | 5 ++ .../hotkey/worker/counter/CounterConfig.java | 7 +- .../worker/counter/CounterConsumer.java | 64 +++++++++++--- .../worker/netty/filter/KeyCounterFilter.java | 12 ++- .../hotkey/worker/starters/InitStarter.java | 4 +- 11 files changed, 253 insertions(+), 78 deletions(-) create mode 100644 client/src/main/java/com/jd/platform/hotkey/client/callback/ValueModel.java diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/AbsReceiveNewKey.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/AbsReceiveNewKey.java index 80b3c56..924532f 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/AbsReceiveNewKey.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/AbsReceiveNewKey.java @@ -25,7 +25,19 @@ public abstract class AbsReceiveNewKey implements ReceiveNewKeyListener { //已经是热key了，又推过来同样的热key，做个日志记录，并刷新一下 if (JdHotKeyStore.isHot(hotKeyModel.getKey())) { JdLogger.warn(getClass(), "receive repeat hot key ：" + hotKeyModel.getKey() + " at " + now); - JdHotKeyStore.setValueDirectly(hotKeyModel.getKey(), JdHotKeyStore.getValueSimple(hotKeyModel.getKey())); + + //可能存在瞬间的该value过期的情况，所以要判空 + ValueModel valueModel = JdHotKeyStore.getValueSimple(hotKeyModel.getKey()); + if (valueModel != null) { + valueModel.setCreateTime(System.currentTimeMillis()); + } else { + //else大概率走不到 + valueModel = ValueModel.defaultValue(hotKeyModel.getKey()); + } + if (valueModel != null) { + //刷新过期时间 + JdHotKeyStore.setValueDirectly(hotKeyModel.getKey(), valueModel); + } } else { //不是重复热key时，新建热key addKey(hotKeyModel.getKey(), hotKeyModel.getKeyType(), hotKeyModel.getCreateTime()); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/DefaultNewKeyListener.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/DefaultNewKeyListener.java index 9751ee0..bd4002f 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/DefaultNewKeyListener.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/DefaultNewKeyListener.java @@ -2,7 +2,6 @@ package com.jd.platform.hotkey.client.callback; import com.jd.platform.hotkey.client.cache.CacheFactory; import com.jd.platform.hotkey.common.model.typeenum.KeyType; -import com.jd.platform.hotkey.common.tool.Constant; /** * @author wuweifeng wrote on 2020-02-24 @@ -13,7 +12,12 @@ public class DefaultNewKeyListener extends AbsReceiveNewKey { @Override void addKey(String key, KeyType keyType, long createTime) { - CacheFactory.getNonNullCache(key).set(key, Constant.MAGIC_NUMBER); + ValueModel valueModel = ValueModel.defaultValue(key); + if (valueModel == null) { + //不符合任何规则 + return; + } + JdHotKeyStore.setValueDirectly(key, valueModel); } @Override diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java index ee43413..65fa35a 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/JdHotKeyStore.java @@ -14,6 +14,17 @@ import com.jd.platform.hotkey.common.tool.Constant; */ public class JdHotKeyStore { + /** + * 是否临近过期 + */ + private static boolean isNearExpire(ValueModel valueModel) { + //判断是否过期时间小于1秒，小于1秒的话也发送 + if (valueModel == null) { + return true; + } + return valueModel.getCreateTime() + valueModel.getDuration() - System.currentTimeMillis() <= 2000; + } + /** * 判断是否是key，如果不是，则发往netty @@ -25,7 +36,14 @@ public class JdHotKeyStore { boolean isHot = isHot(key); if (!isHot) { HotKeyPusher.push(key, null); + } else { + ValueModel valueModel = getValueSimple(key); + //判断是否过期时间小于1秒，小于1秒的话也发送 + if (isNearExpire(valueModel)) { + HotKeyPusher.push(key, null); + } } + //统计计数 KeyHandlerFactory.getCounter().collect(new KeyHotModel(key, isHot)); return isHot; @@ -35,12 +53,16 @@ public class JdHotKeyStore { * 从本地caffeine取值 */ public static Object get(String key) { - Object value = getValueSimple(key); + ValueModel value = getValueSimple(key); + if (value == null) { + return null; + } + Object object = value.getValue(); //如果是默认值也返回null - if(value instanceof Integer && Constant.MAGIC_NUMBER == (int) value) { + if (object instanceof Integer && Constant.MAGIC_NUMBER == (int) object) { return null; } - return value; + return object; } /** @@ -48,7 +70,11 @@ public class JdHotKeyStore { */ public static void smartSet(String key, Object value) { if (isHot(key)) { - setValueDirectly(key, value); + ValueModel valueModel = getValueSimple(key); + if (valueModel == null) { + return; + } + valueModel.setValue(value); } } @@ -60,17 +86,30 @@ public class JdHotKeyStore { if (!inRule(key)) { return null; } - Object value = getValueSimple(key); + Object userValue = null; + + ValueModel value = getValueSimple(key); + if (value == null) { HotKeyPusher.push(key, keyType); + } else { + //临近过期了，也发 + if (isNearExpire(value)) { + HotKeyPusher.push(key, keyType); + } + Object object = value.getValue(); + //如果是默认值，也返回null + if (object instanceof Integer && Constant.MAGIC_NUMBER == (int) object) { + userValue = null; + } else { + userValue = object; + } } + //统计计数 KeyHandlerFactory.getCounter().collect(new KeyHotModel(key, value != null)); - //如果是默认值，也返回null - if(value instanceof Integer && Constant.MAGIC_NUMBER == (int) value) { - return null; - } - return value; + + return userValue; } public static Object getValue(String key) { @@ -80,8 +119,12 @@ public class JdHotKeyStore { /** * 仅获取value，如果不存在也不上报热key */ - static Object getValueSimple(String key) { - return getCache(key).get(key); + static ValueModel getValueSimple(String key) { + Object object = getCache(key).get(key); + if (object == null) { + return null; + } + return (ValueModel) object; } /** @@ -104,25 +147,6 @@ public class JdHotKeyStore { return getValueSimple(key) != null; } - /** - * 判断是否已经缓存过了。 - */ -// private static boolean isValueCached(String key, KeyType keyType) { -// Object value = getValue(key, keyType); -// //如果value不为null且不为默认值 -// if (value == null) { -// return false; -// } -// return !(value instanceof Integer) || Constant.MAGIC_NUMBER != (int) value; -// } - - /** - * 判断某key的value是否已经缓存过了 - */ -// public static boolean isValueCached(String key) { -// return isValueCached(key, KeyType.REDIS_KEY); -// } - private static LocalCache getCache(String key) { return CacheFactory.getNonNullCache(key); } diff --git a/client/src/main/java/com/jd/platform/hotkey/client/callback/ValueModel.java b/client/src/main/java/com/jd/platform/hotkey/client/callback/ValueModel.java new file mode 100644 index 0000000..e5ff17e --- /dev/null +++ b/client/src/main/java/com/jd/platform/hotkey/client/callback/ValueModel.java @@ -0,0 +1,62 @@ +package com.jd.platform.hotkey.client.callback; + +import com.jd.platform.hotkey.client.core.rule.KeyRuleHolder; +import com.jd.platform.hotkey.common.tool.Constant; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-07-07 + */ +public class ValueModel { + /** + * 该热key创建时间 + */ + private long createTime = System.currentTimeMillis(); + /** + * 本地缓存时间，单位毫秒 + */ + private int duration; + /** + * 用户实际存放的value + */ + private Object value; + + public static ValueModel defaultValue(String key) { + ValueModel valueModel = new ValueModel(); + int duration = KeyRuleHolder.duration(key); + if (duration == 0) { + //不符合任何规则 + return null; + } + //转毫秒 + valueModel.setDuration(duration * 1000); + valueModel.setValue(Constant.MAGIC_NUMBER); + return valueModel; + } + + + public long getCreateTime() { + return createTime; + } + + public void setCreateTime(long createTime) { + this.createTime = createTime; + } + + public int getDuration() { + return duration; + } + + public void setDuration(int duration) { + this.duration = duration; + } + + public Object getValue() { + return value; + } + + public void setValue(Object value) { + this.value = value; + } +} diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java index 983b5a8..d5afa72 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/key/NettyKeyPusher.java @@ -1,5 +1,6 @@ package com.jd.platform.hotkey.client.core.key; +import com.jd.platform.hotkey.client.Context; import com.jd.platform.hotkey.client.core.worker.WorkerInfoHolder; import com.jd.platform.hotkey.client.log.JdLogger; import com.jd.platform.hotkey.common.model.HotKeyModel; @@ -76,7 +77,8 @@ public class NettyKeyPusher implements IKeyPusher { for (Channel channel : map.keySet()) { try { List batch = map.get(channel); - channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(MessageType.REQUEST_HIT_COUNT, FastJsonUtils.convertObjectToJSON(batch)))); + channel.writeAndFlush(MsgBuilder.buildByteBuf(new HotKeyMsg(Context.APP_NAME, + MessageType.REQUEST_HIT_COUNT, FastJsonUtils.convertObjectToJSON(batch)))); } catch (Exception e) { try { InetSocketAddress insocket = (InetSocketAddress) channel.remoteAddress(); diff --git a/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java b/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java index 4397d63..37eea64 100644 --- a/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java +++ b/client/src/main/java/com/jd/platform/hotkey/client/core/rule/KeyRuleHolder.java @@ -75,29 +75,11 @@ public class KeyRuleHolder { if (StrUtil.isEmpty(key)) { return null; } - synchronized (KEY_RULES) { - LocalCache prefix = null; - LocalCache common = null; - - //遍历该app的所有rule，找到与key匹配的rule。优先全匹配->prefix匹配-> * 通配 - //这一段虽然看起来比较奇怪，但是没毛病，不要乱改 - for (KeyRule keyRule : KEY_RULES) { - if (key.equals(keyRule.getKey())) { - return RULE_CACHE_MAP.get(keyRule.getDuration()); - } - if ((keyRule.isPrefix() && key.startsWith(keyRule.getKey()))) { - prefix = RULE_CACHE_MAP.get(keyRule.getDuration()); - } - if ("*".equals(keyRule.getKey())) { - common = RULE_CACHE_MAP.get(keyRule.getDuration()); - } - } - - if (prefix != null) { - return prefix; - } - return common; + KeyRule keyRule = findRule(key); + if (keyRule == null) { + return null; } + return RULE_CACHE_MAP.get(keyRule.getDuration()); } @@ -105,13 +87,44 @@ public class KeyRuleHolder { * 判断该key命中了哪个rule */ public static String rule(String key) { - String rule = ""; + KeyRule keyRule = findRule(key); + if (keyRule != null) { + return keyRule.getKey(); + } + return ""; + } + + /** + * 获取该key应该缓存多久 + */ + public static int duration(String key) { + KeyRule keyRule = findRule(key); + if (keyRule != null) { + return keyRule.getDuration(); + } + return 0; + } + + //遍历该app的所有rule，找到与key匹配的rule。优先全匹配->prefix匹配-> * 通配 + //这一段虽然看起来比较奇怪，但是没毛病，不要乱改 + private static KeyRule findRule(String key) { + KeyRule prefix = null; + KeyRule common = null; for (KeyRule keyRule : KEY_RULES) { - if (key.equals(keyRule.getKey()) || (keyRule.isPrefix() && key.startsWith(keyRule.getKey())) || "*".equals(keyRule.getKey())) { - return keyRule.getKey(); + if (key.equals(keyRule.getKey())) { + return keyRule; + } + if ((keyRule.isPrefix() && key.startsWith(keyRule.getKey()))) { + prefix = keyRule; } + if ("*".equals(keyRule.getKey())) { + common = keyRule; + } + } + if (prefix != null) { + return prefix; } - return rule; + return common; } /** diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java index 9ac0b37..e88389d 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/ConfigConstant.java @@ -49,4 +49,9 @@ public interface ConfigConstant { * bufferPool直接内存 */ String bufferPoolPath = "/jd/bufferPool/"; + + /** + * 存放客户端hotKey访问次数和总访问次数的path + */ + String keyHitCountPath = "/jd/keyHitCount/"; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConfig.java index 6fb058b..c0d09b2 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConfig.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConfig.java @@ -1,8 +1,10 @@ package com.jd.platform.hotkey.worker.counter; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import javax.annotation.Resource; import java.util.concurrent.DelayQueue; /** @@ -17,10 +19,13 @@ public class CounterConfig { */ public static DelayQueue DELAY_QUEUE = new DelayQueue<>(); + @Resource + private IConfigCenter configCenter; + @Bean public CounterConsumer counterConsumer() { CounterConsumer counterConsumer = new CounterConsumer(); - counterConsumer.beginConsume(); + counterConsumer.beginConsume(configCenter); return counterConsumer; } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConsumer.java index 52218f6..71672db 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConsumer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConsumer.java @@ -1,8 +1,18 @@ package com.jd.platform.hotkey.worker.counter; +import com.jd.platform.hotkey.common.configcenter.ConfigConstant; +import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.model.KeyCountModel; +import com.jd.platform.hotkey.common.tool.Constant; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.common.tool.IpUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; import static com.jd.platform.hotkey.worker.counter.CounterConfig.DELAY_QUEUE; @@ -12,24 +22,52 @@ import static com.jd.platform.hotkey.worker.counter.CounterConfig.DELAY_QUEUE; * @date 2020-06-28 */ public class CounterConsumer { + private Logger logger = LoggerFactory.getLogger(getClass()); - public void beginConsume() { - while (true) { - try { - KeyCountItem item = DELAY_QUEUE.take(); - List keyCountModels = item.getList(); - String appName = item.getAppName(); - for (KeyCountModel keyCountModel : keyCountModels) { - String ruleKey = keyCountModel.getRuleKey(); - int totalHitCount = keyCountModel.getTotalHitCount(); - int hotHitCount = keyCountModel.getHotHitCount(); + public void beginConsume(IConfigCenter configCenter) { + CompletableFuture.runAsync(() -> { + Map map = new HashMap<>(500); + + while (true) { + try { + KeyCountItem item = DELAY_QUEUE.take(); + //每个List是一个client的10秒内的数据，一个rule如果每秒都有数据，那list里就有10条 + List keyCountModels = item.getList(); + String appName = item.getAppName(); + for (KeyCountModel keyCountModel : keyCountModels) { + //如 rule + Constant.COUNT_DELIMITER + nowTime; + //rule + 分隔符 + 2020-10-23 21:11:22 + //pin__#**#2020-10-23 21:11:22 + String ruleKey = keyCountModel.getRuleKey(); + int hotHitCount = keyCountModel.getHotHitCount(); + int totalHitCount = keyCountModel.getTotalHitCount(); + //key：ConfigConstant.keyHitCountPath + appName + "/" + IpUtils.getIp() + "-" + System.currentTimeMillis() + String mapKey = appName + Constant.COUNT_DELIMITER + ruleKey; + if (map.get(mapKey) == null) { + map.put(mapKey, hotHitCount + "-" + totalHitCount); + } else { + String[] counts = map.get(mapKey).split("-"); + int hotCount = Integer.valueOf(counts[0]) + hotHitCount; + int totalCount = Integer.valueOf(counts[1]) + totalHitCount; + map.put(mapKey, hotCount + "-" + totalCount); + } + } + //500是什么意思呢？300就代表了300秒的数据了，已经不少了 + if (map.size() >= 300) { + configCenter.putAndGrant(ConfigConstant.keyHitCountPath + appName + "/" + IpUtils.getIp() + + "-" + System.currentTimeMillis(), + FastJsonUtils.convertObjectToJSON(map), 30); + logger.info("key Hit count : " + map); + map.clear(); + } + + } catch (InterruptedException e) { + e.printStackTrace(); } - } catch (InterruptedException e) { - e.printStackTrace(); } + }); - } } } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java index b095dee..78ad416 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java @@ -1,6 +1,7 @@ package com.jd.platform.hotkey.worker.netty.filter; import cn.hutool.core.date.SystemClock; +import cn.hutool.core.util.StrUtil; import com.jd.platform.hotkey.common.model.HotKeyMsg; import com.jd.platform.hotkey.common.model.KeyCountModel; import com.jd.platform.hotkey.common.model.typeenum.MessageType; @@ -12,6 +13,7 @@ import com.jd.platform.hotkey.worker.tool.InitConstant; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.core.annotation.Order; import org.springframework.stereotype.Component; @@ -29,11 +31,19 @@ import static com.jd.platform.hotkey.worker.counter.CounterConfig.DELAY_QUEUE; public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { private Logger logger = LoggerFactory.getLogger(getClass()); + /** + * 该worker放到etcd worker目录的哪个app下 + */ + @Value("${etcd.workerPath}") + private String workerPath; @Override public boolean chain(HotKeyMsg message, ChannelHandlerContext ctx) { if (MessageType.REQUEST_HIT_COUNT == message.getMessageType()) { - + //设置appName + if (StrUtil.isEmpty(message.getAppName())) { + message.setAppName(workerPath); + } publishMsg(message.getAppName(), message.getBody(), ctx); return false; diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/InitStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/InitStarter.java index 42f8fa1..53c9f2c 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/InitStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/InitStarter.java @@ -3,14 +3,14 @@ package com.jd.platform.hotkey.worker.starters; import com.jd.platform.hotkey.worker.tool.InitConstant; import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; +import org.springframework.stereotype.Component; /** * @author wuweifeng * @version 1.0 * @date 2020-05-22 */ -@Configuration +@Component public class InitStarter { @Value("${netty.timeOut}") private int timeOut; -- Gitee From 60c2e070d81a4b8116e620e8e5a8601fe61037f2 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Wed, 8 Jul 2020 16:27:58 +0800 Subject: [PATCH 34/51] =?UTF-8?q?netty=E7=89=88=E6=9C=AC=E7=BB=9F=E4=B8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/pom.xml b/common/pom.xml index 4c855aa..59a7d76 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -14,7 +14,7 @@ 1.8 - 4.1.24.Final + 4.1.42.Final 3.4.2 1.1.7.3 1.2.70 -- Gitee From 5195965ade2c1694276169f81e12ed07beb0f0da Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Fri, 10 Jul 2020 12:09:44 +0800 Subject: [PATCH 35/51] update README.md. --- README.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/README.md b/README.md index a4a60f2..f6905bd 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,25 @@ public void initHotkey() { ``` 其中还可以setCaffeineSize(int size)设置本地缓存最大数量，默认5万，setPushPeriod(Long period)设置批量推送key的间隔时间，默认500ms，该值越小，上报热key越频繁，相应越及时，建议根据实际情况调整，如单机每秒qps10个，那么0.5秒上报一次即可，否则是空跑。该值最小为1，即1ms上报一次。 +注意： + +如果原有项目里使用了guava，需要升级guava为以下版本，否则过低的guava版本可能发生jar包冲突。或者删除自己项目里的guava的maven依赖，guava升级不会影响原有任何逻辑。 + +``` + + com.google.guava + guava + 28.2-jre + compile + +``` + +有时可能项目里没有直接依赖guava，但是引入的某个pom里引了guava，也需要将guava排除掉。 + + + +使用： + 主要有如下4个方法可供使用 boolean JdHotKeyStore.isHotKey(String key) -- Gitee From 60ab57b458baa10348bf1010768671df3e176795 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 10 Jul 2020 16:56:30 +0800 Subject: [PATCH 36/51] 1 --- .../configcenter/etcd/JdEtcdClient.java | 9 ++- .../platform/hotkey/common/tool/Constant.java | 2 + .../dashboard/common/base/BaseController.java | 20 ++++- .../dashboard/common/domain/Constant.java | 3 +- .../dashboard/common/monitor/DataHandler.java | 54 ++++++------- .../dashboard/common/monitor/EtcdMonitor.java | 80 ++++++++++++------- .../dashboard/controller/KeyController.java | 6 +- .../hotkey/worker/counter/CounterConfig.java | 4 +- .../worker/counter/CounterConsumer.java | 10 +-- .../worker/netty/filter/KeyCounterFilter.java | 8 +- .../hotkey/worker/starters/EtcdStarter.java | 47 +++++++---- .../worker/starters/NodesServerStarter.java | 4 +- .../hotkey/worker/tool/AsyncPool.java | 21 +++++ .../hotkey/worker/tool/MemoryTool.java | 14 ++++ 14 files changed, 189 insertions(+), 93 deletions(-) create mode 100644 worker/src/main/java/com/jd/platform/hotkey/worker/tool/AsyncPool.java diff --git a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java index b6ddd58..fb64d9e 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/configcenter/etcd/JdEtcdClient.java @@ -3,7 +3,6 @@ package com.jd.platform.hotkey.common.configcenter.etcd; import cn.hutool.core.collection.CollectionUtil; import com.google.protobuf.ByteString; import com.ibm.etcd.api.KeyValue; -import com.ibm.etcd.api.Kv; import com.ibm.etcd.api.LeaseGrantResponse; import com.ibm.etcd.api.RangeResponse; import com.ibm.etcd.client.KvStoreClient; @@ -54,6 +53,14 @@ public class JdEtcdClient implements IConfigCenter { this.kvClient = kvClient; } + public LockClient getLockClient() { + return lockClient; + } + + public void setLockClient(LockClient lockClient) { + this.lockClient = lockClient; + } + @Override public void put(String key, String value) { kvClient.put(ByteString.copyFromUtf8(key), ByteString.copyFromUtf8(value)).sync(); diff --git a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java index 0c9bf86..1e0765b 100644 --- a/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java +++ b/common/src/main/java/com/jd/platform/hotkey/common/tool/Constant.java @@ -23,6 +23,8 @@ public class Constant { */ public static String COUNT_DELIMITER = "#**#"; + public static String BAK_DELIMITER = "#\\*\\*#"; + //单次包最大2M public static int MAX_LENGTH = 2 * 1024 * 1024; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java index c7dd2d4..91555ff 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java @@ -1,26 +1,32 @@ package com.jd.platform.hotkey.dashboard.common.base; +import java.text.SimpleDateFormat; +import java.util.Date; + import com.alibaba.fastjson.JSON; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; +import com.jd.platform.hotkey.dashboard.model.User; +import com.jd.platform.hotkey.dashboard.service.UserService; import com.jd.platform.hotkey.dashboard.util.JwtTokenUtil; import io.jsonwebtoken.Claims; import org.springframework.beans.propertyeditors.CustomDateEditor; + import org.springframework.web.bind.WebDataBinder; import org.springframework.web.bind.annotation.InitBinder; import javax.annotation.Resource; import javax.servlet.http.HttpServletRequest; -import java.text.SimpleDateFormat; -import java.util.Date; public class BaseController { @Resource protected HttpServletRequest request; + @Resource + protected UserService userService; @InitBinder public void initBinder(WebDataBinder binder) { @@ -35,8 +41,7 @@ public class BaseController { Claims claims = JwtTokenUtil.claims(authHeader.substring(2)); String role = claims.get("role",String.class); if(!role.equals(Constant.ADMIN)){ - String appName = claims.get("appName",String.class); - if(!appName.equals(app)){ + if(!appName().equals(app)){ throw new BizException(ResultEnum.NO_PERMISSION); } } @@ -60,4 +65,11 @@ public class BaseController { } + public String appName(){ + String authHeader = JwtTokenUtil.getAuthHeader(request); + assert authHeader != null; + Claims claims = JwtTokenUtil.claims(authHeader.substring(2)); + return userService.selectByUserName(claims.getSubject()).getAppName(); + } + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java index de02283..d9b8101 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/Constant.java @@ -1,7 +1,6 @@ package com.jd.platform.hotkey.dashboard.common.domain; import java.util.ArrayList; -import java.util.Collections; import java.util.List; public class Constant { @@ -20,6 +19,8 @@ public class Constant { public static final int WORKER_CHANGE = 2; + public static final int HOTKEY_CHANGE = 3; + public static final String CHANG_LOG_VIEW = "变更记录"; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index 3027d39..c98c427 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -25,7 +25,6 @@ import java.time.LocalDateTime; import java.util.Date; import java.util.List; import java.util.concurrent.BlockingQueue; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.LinkedBlockingQueue; @Component @@ -57,36 +56,34 @@ public class DataHandler { } public void insertRecords() { - CompletableFuture.runAsync(() -> { - while (true) { - TwoTuple twoTuple; + while (true) { + TwoTuple twoTuple; + try { + twoTuple = handHotKey(queue.take()); + } catch (Exception e) { + e.printStackTrace(); + log.error("handHotKey error ," + e.getCause()); + continue; + } + KeyRecord keyRecord = twoTuple.getSecond(); + KeyTimely keyTimely = twoTuple.getFirst(); + + if (keyTimely.getUuid() == null) { + keyTimelyMapper.deleteByKeyAndApp(keyTimely.getKey(), keyTimely.getAppName()); + } else { try { - twoTuple = handHotKey(queue.take()); + keyTimelyMapper.insertSelective(keyTimely); } catch (Exception e) { - e.printStackTrace(); - log.error("handHotKey error ," + e.getCause()); - continue; - } - KeyRecord keyRecord = twoTuple.getSecond(); - KeyTimely keyTimely = twoTuple.getFirst(); - - if (keyTimely.getUuid() == null) { - keyTimelyMapper.deleteByKeyAndApp(keyTimely.getKey(), keyTimely.getAppName()); - } else { - try { - keyTimelyMapper.insertSelective(keyTimely); - } catch (Exception e) { - log.info("insert timely error"); - } - } - - if (keyRecord != null) { - //插入记录 - keyRecordMapper.insertSelective(keyRecord); + log.info("insert timely error"); } + } + if (keyRecord != null) { + //插入记录 + keyRecordMapper.insertSelective(keyRecord); } - }); + + } } @@ -128,7 +125,6 @@ public class DataHandler { } - /** * 每小时 统计一次record 表 结果记录到统计表 */ @@ -164,7 +160,9 @@ public class DataHandler { log.info("每小时统计规则,时间：{}, data list：{}", now.toString(), JSON.toJSONString(statistics)); records.addAll(statistics); } - int row = statisticsMapper.batchInsert(records); + if (records.size() > 0) { + int row = statisticsMapper.batchInsert(records); + } } catch (Exception e) { e.printStackTrace(); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java index 4fc14d9..9801701 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java @@ -4,15 +4,14 @@ import cn.hutool.core.util.StrUtil; import com.ibm.etcd.api.Event; import com.ibm.etcd.api.KeyValue; import com.ibm.etcd.client.kv.KvClient; -import com.ibm.etcd.client.kv.WatchUpdate; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; -import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.mapper.ReceiveCountMapper; +import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.ReceiveCount; import com.jd.platform.hotkey.dashboard.model.Worker; import com.jd.platform.hotkey.dashboard.service.WorkerService; @@ -27,7 +26,9 @@ import javax.annotation.Resource; import java.util.ArrayList; import java.util.Date; import java.util.List; -import java.util.concurrent.CompletableFuture; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; /** * @ProjectName: hotkey @@ -43,36 +44,26 @@ public class EtcdMonitor { @Resource private IConfigCenter configCenter; - @Resource - private ChangeLogMapper logMapper; + @Resource private WorkerService workerService; + @Resource + private SummaryMapper summaryMapper; + @Resource private DataHandler dataHandler; @Resource private ReceiveCountMapper receiveCountMapper; -// @PostConstruct -// public void init() { -// CompletableFuture.runAsync(() -> { -// try { -// Thread.sleep(100); -// } catch (InterruptedException e) { -// e.printStackTrace(); -// } -// for (int i = 0; i < 200; i++) { -// configCenter.put(ConfigConstant.hotKeyRecordPath + "abc/" + i, UUID.randomUUID().toString()); -// } -// }); -// } + public static final ExecutorService threadPoolExecutor = Executors.newCachedThreadPool(); /** * 监听新来的热key，该key的产生是来自于手工在控制台添加 */ public void watchHandOperationHotKey() { - CompletableFuture.runAsync(() -> { + threadPoolExecutor.submit(() -> { KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyPath); while (watchIterator.hasNext()) { Event event = event(watchIterator); @@ -89,7 +80,7 @@ public class EtcdMonitor { * 监听新来的热key，该key的产生是来自于worker集群推送过来的 */ public void watchHotKeyRecord() { - CompletableFuture.runAsync(() -> { + threadPoolExecutor.submit(() -> { KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.hotKeyRecordPath); while (watchIterator.hasNext()) { Event event = event(watchIterator); @@ -125,7 +116,7 @@ public class EtcdMonitor { fetchRuleFromEtcd(); //规则拉取完毕后才能去开始入库 - dataHandler.insertRecords(); + insertRecords(); //开始监听热key产生 watchHotKeyRecord(); @@ -136,22 +127,30 @@ public class EtcdMonitor { watchRule(); watchWorkers(); + + //观察热key访问次数和总访问次数，并做统计 + watchHitCount(); + } + + private void insertRecords() { + threadPoolExecutor.submit(() -> { + dataHandler.insertRecords(); + }); } /** * 异步监听rule规则变化 */ - - public void watchRule() { - CompletableFuture.runAsync(() -> { + private void watchRule() { + threadPoolExecutor.submit(() -> { try { KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.rulePath); //如果有新事件，即rule的变更，就重新拉取所有的信息 while (watchIterator.hasNext()) { //这句必须写，next会让他卡住，除非真的有新rule变更 - WatchUpdate watchUpdate = watchIterator.next(); - List eventList = watchUpdate.getEvents(); + Event event = event(watchIterator); + log.info("---------watch rule change---------"); //全量拉取rule信息 fetchRuleFromEtcd(); } @@ -197,7 +196,7 @@ public class EtcdMonitor { } private void watchWorkers() { - CompletableFuture.runAsync(() -> { + threadPoolExecutor.submit(() -> { KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.workersPath); while (watchIterator.hasNext()) { Event event = event(watchIterator); @@ -218,9 +217,8 @@ public class EtcdMonitor { }); } - //@PostConstruct - public void watchReceiveKeyCount() { - CompletableFuture.runAsync(() -> { + private void watchReceiveKeyCount() { + threadPoolExecutor.submit(() -> { KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.totalReceiveKeyCount); while (watchIterator.hasNext()) { Event event = event(watchIterator); @@ -239,6 +237,28 @@ public class EtcdMonitor { }); } + /** + * 监听热key访问次数和总访问次数 + */ + private void watchHitCount() { + threadPoolExecutor.submit(() -> { + //key：ConfigConstant.keyHitCountPath + appName + "/" + IpUtils.getIp() + "-" + System.currentTimeMillis() + //value：FastJsonUtils.convertObjectToJSON(map) + //map的key是 appName + #**# + pin__#**#2020-10-23 21:11:22 + //map的value是"67-1937" 前面是热key访问量，后面是总访问量 + KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.keyHitCountPath); + while (watchIterator.hasNext()) { + Event event = event(watchIterator); + KeyValue kv = event.getKv(); + String k = kv.getKey().toStringUtf8(); + String v = kv.getValue().toStringUtf8(); + Map map = FastJsonUtils.stringToCollect(v); + for (String key : map.keySet()) { +// int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map)); + } + } + }); + } private Event event(KvClient.WatchIterator watchIterator) { return watchIterator.next().getEvents().get(0); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java index e0c54e3..bac86cc 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java @@ -44,7 +44,8 @@ public class KeyController extends BaseController { @PostMapping("/ruleLineChart") @ResponseBody public HotKeyLineChartVo ruleLineChart(SearchReq req){ - return keyService.ruleLineChart(req); + String app = appName(); + return keyService.ruleLineChart(req,app); } @@ -117,6 +118,7 @@ public class KeyController extends BaseController { @ResponseBody public Result add(KeyTimely key){ checkApp(key.getAppName()); + key.setUpdater(userName()); int b = keyService.insertKeyByUser(key); return b == 0 ? Result.fail():Result.success(); } @@ -126,7 +128,7 @@ public class KeyController extends BaseController { public Result remove(String key){ String[] arr = key.split("/"); checkApp(arr[0]); - int b = keyService.delKeyByUser(new KeyTimely(key)); + int b = keyService.delKeyByUser(new KeyTimely(key,userName())); return b == 0 ? Result.fail():Result.success(); } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConfig.java b/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConfig.java index c0d09b2..c9a4f41 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConfig.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConfig.java @@ -5,7 +5,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import javax.annotation.Resource; -import java.util.concurrent.DelayQueue; +import java.util.concurrent.LinkedBlockingQueue; /** * @author wuweifeng @@ -17,7 +17,7 @@ public class CounterConfig { /** * 队列 */ - public static DelayQueue DELAY_QUEUE = new DelayQueue<>(); + public static LinkedBlockingQueue COUNTER_QUEUE = new LinkedBlockingQueue<>(); @Resource private IConfigCenter configCenter; diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConsumer.java b/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConsumer.java index 71672db..56ae146 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConsumer.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/counter/CounterConsumer.java @@ -6,15 +6,15 @@ import com.jd.platform.hotkey.common.model.KeyCountModel; import com.jd.platform.hotkey.common.tool.Constant; import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.common.tool.IpUtils; +import com.jd.platform.hotkey.worker.tool.AsyncPool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; -import static com.jd.platform.hotkey.worker.counter.CounterConfig.DELAY_QUEUE; +import static com.jd.platform.hotkey.worker.counter.CounterConfig.COUNTER_QUEUE; /** * @author wuweifeng @@ -25,13 +25,11 @@ public class CounterConsumer { private Logger logger = LoggerFactory.getLogger(getClass()); public void beginConsume(IConfigCenter configCenter) { - CompletableFuture.runAsync(() -> { - + AsyncPool.asyncDo(() -> { Map map = new HashMap<>(500); - while (true) { try { - KeyCountItem item = DELAY_QUEUE.take(); + KeyCountItem item = COUNTER_QUEUE.take(); //每个List是一个client的10秒内的数据，一个rule如果每秒都有数据，那list里就有10条 List keyCountModels = item.getList(); String appName = item.getAppName(); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java index 78ad416..d8eae48 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/netty/filter/KeyCounterFilter.java @@ -19,7 +19,7 @@ import org.springframework.stereotype.Component; import java.util.List; -import static com.jd.platform.hotkey.worker.counter.CounterConfig.DELAY_QUEUE; +import static com.jd.platform.hotkey.worker.counter.CounterConfig.COUNTER_QUEUE; /** * 对热key访问次数和总访问次数进行累计 @@ -67,7 +67,11 @@ public class KeyCounterFilter implements INettyMsgFilter, IMqMessageReceiver { return; } //将收到的key放入延时队列，15秒后进行累加并发送 - DELAY_QUEUE.put(new KeyCountItem(appName, models.get(0).getCreateTime(), models)); + try { + COUNTER_QUEUE.put(new KeyCountItem(appName, models.get(0).getCreateTime(), models)); + } catch (InterruptedException e) { + e.printStackTrace(); + } } } \ No newline at end of file diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java index 81b8ccb..bf9f6fa 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/EtcdStarter.java @@ -17,6 +17,7 @@ import com.jd.platform.hotkey.worker.netty.filter.HotKeyFilter; import com.jd.platform.hotkey.worker.netty.holder.ClientInfoHolder; import com.jd.platform.hotkey.worker.netty.holder.WhiteListHolder; import com.jd.platform.hotkey.worker.rule.KeyRuleHolder; +import com.jd.platform.hotkey.worker.tool.AsyncPool; import io.grpc.StatusRuntimeException; import io.netty.channel.ChannelHandlerContext; import org.slf4j.Logger; @@ -24,14 +25,12 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -import org.springframework.util.CollectionUtils; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import javax.annotation.Resource; import java.util.List; import java.util.Map; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; @@ -60,6 +59,8 @@ public class EtcdStarter { @Value("${etcd.workerPath}") private String workerPath; + private static final String DEFAULT_PATH = "default"; + private static final String MAO = ":"; private static final String ETCD_DOWN = "etcd is unConnected . please do something"; private static final String EMPTY_RULE = "very important warn !!! rule info is null!!!"; @@ -72,13 +73,25 @@ public class EtcdStarter { //KeepAliveOnce：为某个租约续约一次。 //Close：貌似是关闭当前客户端建立的所有租约。 + /** + * 该worker是否只服务于一个应用 + */ + private boolean isForSingle() { + return !DEFAULT_PATH.equals(workerPath); + } + /** * 启动回调监听器，监听rule变化 */ @PostConstruct public void watch() { - CompletableFuture.runAsync(() -> { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.rulePath); + AsyncPool.asyncDo(() -> { + KvClient.WatchIterator watchIterator; + if (isForSingle()) { + watchIterator = configCenter.watch(ConfigConstant.rulePath + workerPath); + } else { + watchIterator = configCenter.watchPrefix(ConfigConstant.rulePath); + } while (watchIterator.hasNext()) { WatchUpdate watchUpdate = watchIterator.next(); List eventList = watchUpdate.getEvents(); @@ -97,7 +110,7 @@ public class EtcdStarter { */ @PostConstruct public void watchWhiteList() { - CompletableFuture.runAsync(() -> { + AsyncPool.asyncDo(() -> { //获取所有白名单 fetchWhite(); @@ -127,20 +140,23 @@ public class EtcdStarter { */ @Scheduled(fixedRate = 60000) public void pullRules() { - List keyValues; try { - keyValues = configCenter.getPrefix(ConfigConstant.rulePath); + if (isForSingle()) { + String value = configCenter.get(ConfigConstant.rulePath + workerPath); + if (!StrUtil.isEmpty(value)) { + List keyRules = FastJsonUtils.toList(value, KeyRule.class); + KeyRuleHolder.put(workerPath, keyRules); + } + } else { + List keyValues = configCenter.getPrefix(ConfigConstant.rulePath); + for (KeyValue keyValue : keyValues) { + ruleChange(keyValue); + } + } } catch (StatusRuntimeException ex) { logger.error(ETCD_DOWN); - return; - } - if (CollectionUtils.isEmpty(keyValues)) { - logger.warn(EMPTY_RULE); - return; - } - for (KeyValue keyValue : keyValues) { - ruleChange(keyValue); } + } /** @@ -189,6 +205,7 @@ public class EtcdStarter { try { String hostName = IpUtils.getHostName(); configCenter.delete(ConfigConstant.workersPath + hostName); + AsyncPool.shutDown(); } catch (Exception e) { logger.error("worker connect to etcd failure"); } diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java index 94f8c8a..0b9c522 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/starters/NodesServerStarter.java @@ -4,6 +4,7 @@ import com.jd.platform.hotkey.common.coder.Codec; import com.jd.platform.hotkey.worker.netty.client.IClientChangeListener; import com.jd.platform.hotkey.worker.netty.filter.INettyMsgFilter; import com.jd.platform.hotkey.worker.netty.server.NodesServer; +import com.jd.platform.hotkey.worker.tool.AsyncPool; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Value; @@ -12,7 +13,6 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; import java.util.List; -import java.util.concurrent.CompletableFuture; /** * @author wuweifeng wrote on 2019-12-11 @@ -34,7 +34,7 @@ public class NodesServerStarter { @PostConstruct public void start() { - CompletableFuture.runAsync(() -> { + AsyncPool.asyncDo(() -> { logger.info("netty server is starting"); NodesServer nodesServer = new NodesServer(); diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/AsyncPool.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/AsyncPool.java new file mode 100644 index 0000000..10b62b7 --- /dev/null +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/AsyncPool.java @@ -0,0 +1,21 @@ +package com.jd.platform.hotkey.worker.tool; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +/** + * @author wuweifeng + * @version 1.0 + * @date 2020-07-10 + */ +public class AsyncPool { + private static ExecutorService threadPoolExecutor = Executors.newCachedThreadPool(); + + public static void asyncDo(Runnable runnable) { + threadPoolExecutor.submit(runnable); + } + + public static void shutDown() { + threadPoolExecutor.shutdown(); + } +} diff --git a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/MemoryTool.java b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/MemoryTool.java index ee2cdc3..66dd70f 100644 --- a/worker/src/main/java/com/jd/platform/hotkey/worker/tool/MemoryTool.java +++ b/worker/src/main/java/com/jd/platform/hotkey/worker/tool/MemoryTool.java @@ -1,11 +1,15 @@ package com.jd.platform.hotkey.worker.tool; +import jdk.nashorn.internal.ir.debug.ObjectSizeCalculator; import sun.misc.JavaNioAccess; import sun.misc.SharedSecrets; import java.lang.management.BufferPoolMXBean; import java.lang.management.ManagementFactory; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.UUID; /** * @author wuweifeng @@ -38,4 +42,14 @@ public class MemoryTool { public static JavaNioAccess.BufferPool getNioBufferPool(){ return SharedSecrets.getJavaNioAccess().getDirectBufferPool(); } + + public static void main(String[] args) { + Map map = new HashMap<>(); + for (int i = 0; i < 1000; i++) { + + map.put(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + } + double m = ObjectSizeCalculator.getObjectSize(map); + System.out.println(m); + } } -- Gitee From 2c2c67b4620318b0e3ae75658a4f760bbddcd47e Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Fri, 10 Jul 2020 17:03:15 +0800 Subject: [PATCH 37/51] Merge branch 'master_erp' # Conflicts: # dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpProperties.java # dashboard/src/main/java/com/jd/platform/hotkey/dashboard/erp/ErpUimInterceptor.java # dashboard/src/main/resources/db.sql --- dashboard/src/main/resources/application.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index b910068..396fc8c 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -1,5 +1,5 @@ server : - port : 8080 + port : 8081 servlet : context-path : / spring : @@ -36,9 +36,9 @@ pagehelper: reasonable: true supportMethodsArguments: true params: count=countSql -mybatis: - configuration: - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl etcd: server: ${etcdServer:http://127.0.0.1:2379} -- Gitee From f6ed6c2ede0291f88de5807eed708037610d512a Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Fri, 10 Jul 2020 19:18:26 +0800 Subject: [PATCH 38/51] Merge branch 'master' of D:\project2\hotkey with conflicts. --- .../dashboard/DashboardApplication.java | 47 ++++++++++++++++++- .../dashboard/common/monitor/EtcdMonitor.java | 10 ++-- dashboard/src/main/resources/application.yml | 19 +++++--- dashboard/src/main/resources/db.sql | 2 +- 4 files changed, 65 insertions(+), 13 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 2b361c6..1c295c2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -4,9 +4,12 @@ package com.jd.platform.hotkey.dashboard; import com.alibaba.fastjson.JSON; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; +import com.jd.platform.hotkey.common.tool.FastJsonUtils; import com.jd.platform.hotkey.common.tool.IpUtils; import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; import com.jd.platform.hotkey.dashboard.mapper.RulesMapper; +import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; +import com.jd.platform.hotkey.dashboard.util.CommonUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; @@ -27,7 +30,8 @@ public class DashboardApplication implements CommandLineRunner { private Logger logger = LoggerFactory.getLogger(getClass()); - + @Resource + private SummaryMapper summaryMapper; @Resource private KeyTimelyMapper timelyMapper; @@ -42,6 +46,47 @@ public class DashboardApplication implements CommandLineRunner { @Override public void run(String... args) { + /* Map map1 = new HashMap<>(); + String k1 = "cartsoa#**#rule1__#**#2020-10-23 21:11:21"; + String v1 = "50-200"; + String k2 = "cartsoa#**#rule2__#**#2020-10-23 21:11:22"; + String v2 = "30-100"; + map1.put(k1,v1); + map1.put(k2,v2); + for (String key : map1.keySet()) { + int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map1)); + } + + Map map12 = new HashMap<>(); + String k12 = "cartsoa#**#rule1__#**#2020-10-23 21:11:21"; + String v12 = "10-100"; + String k22 = "cartsoa#**#rule2__#**#2020-10-23 21:11:23"; + String v22 = "70-300"; + String kk22 = "cartsoa#**#rule2__#**#2020-10-23 21:11:22"; + String vv22 = "87-500"; + String k33 = "cartsoa#**#rule3__#**#2020-10-23 21:11:22"; + String v33 = "72-300"; + map12.put(k12,v12); + map12.put(k22,v22); + map12.put(k33,v33); + map12.put(kk22,vv22); + for (String key : map12.keySet()) { + int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map12)); + } + + Map map122 = new HashMap<>(); + String k122 = "cartsoa#**#rule1__#**#2020-10-23 21:11:23"; + String v122 = "44-55"; + String k222 = "cartsoa#**#rule2__#**#2020-10-23 21:11:25"; + String v222 = "66-77"; + String k332 = "cartsoa#**#rule3__#**#2020-10-23 21:11:27"; + String v332 = "88-99"; + map122.put(k122,v122); + map122.put(k222,v222); + map122.put(k332,v332); + for (String key : map122.keySet()) { + int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map122)); + }*/ int row = timelyMapper.clear(); logger.info("clear db timely hotKey, effect row : {}",row); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java index 9801701..27369d5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java @@ -8,6 +8,7 @@ import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.common.tool.FastJsonUtils; +import com.jd.platform.hotkey.common.tool.IpUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; import com.jd.platform.hotkey.dashboard.mapper.ReceiveCountMapper; @@ -15,6 +16,7 @@ import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; import com.jd.platform.hotkey.dashboard.model.ReceiveCount; import com.jd.platform.hotkey.dashboard.model.Worker; import com.jd.platform.hotkey.dashboard.service.WorkerService; +import com.jd.platform.hotkey.dashboard.util.CommonUtil; import com.jd.platform.hotkey.dashboard.util.RuleUtil; import io.grpc.StatusRuntimeException; import org.slf4j.Logger; @@ -23,10 +25,7 @@ import org.springframework.stereotype.Component; import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -253,8 +252,9 @@ public class EtcdMonitor { String k = kv.getKey().toStringUtf8(); String v = kv.getValue().toStringUtf8(); Map map = FastJsonUtils.stringToCollect(v); + for (String key : map.keySet()) { -// int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map)); + int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map)); } } }); diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index 396fc8c..bc4609f 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -1,5 +1,5 @@ server : - port : 8081 + port : 8088 servlet : context-path : / spring : @@ -29,17 +29,24 @@ spring : datasource: username: root password: JRTEST - url: jdbc:mysql://172.24.7.182:3306/test1?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT%2B8 + url: jdbc:mysql://172.24.7.182:3306/test1?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC driver-class-name: com.mysql.cj.jdbc.Driver pagehelper: helperDialect: mysql reasonable: true supportMethodsArguments: true params: count=countSql -#mybatis: -# configuration: -# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl +mybatis: + configuration: + log-impl: org.apache.ibatis.logging.stdout.StdOutImpl etcd: server: ${etcdServer:http://127.0.0.1:2379} - +erp: + enabled: false + ssoAppKey: fangzhou + ssoAppToken: 3f48d7ade61940f2ba22079ebc273fb8 + loginUrl: https://ssa.jd.com/sso/login + excludePath: /login,/assets,/bootstarp,/common,/htmlformat,/images,/jn,/jquery,/user/login,login + ssoAppUrl: http://ssa.jd.com + defaultPwd: 123456 diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index 2f128a7..6dac5e4 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -175,7 +175,7 @@ CREATE TABLE `hk_summary` ( PRIMARY KEY (`id`) USING BTREE, UNIQUE INDEX `uniq_uuid`(`uuid`) USING BTREE COMMENT '防重索引', INDEX `idx_apprule`(`app`, `rule`) USING BTREE COMMENT '查询索引', - INDEX `ix_ct`(`create_time`) USING BTREE COMMENT '事件索引' + INDEX `ix_ct`(`create_time`) USING BTREE COMMENT '时间索引' ) ENGINE = InnoDB AUTO_INCREMENT = 18 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin COMMENT = '汇总表' ROW_FORMAT = Compact; -- Gitee From 53f48d8ca6d89a5740a455397608f465e8ef9b61 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Mon, 13 Jul 2020 13:14:39 +0800 Subject: [PATCH 39/51] update README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f6905bd..1a7808a 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ https://mp.weixin.qq.com/s/xOzEj5HtCeh_ezHDPHw6Jw ### 界面效果 ![输入图片说明](https://images.gitee.com/uploads/images/2020/0622/163805_0aa68d4b_303698.png "屏幕截图.png") ### 微信群 -![输入图片说明](https://images.gitee.com/uploads/images/2020/0706/095712_fc4d8d97_303698.jpeg "11.jpeg") +![输入图片说明](https://images.gitee.com/uploads/images/2020/0713/131433_c172945c_303698.jpeg "11.jpeg") #### 安装教程 -- Gitee From 756baf23f7c3405975bc028bfea278264f89d9f6 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 14 Jul 2020 22:02:25 +0800 Subject: [PATCH 40/51] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E5=92=8CSQL=E8=A1=A8=20=E5=A2=9E=E5=8A=A0?= =?UTF-8?q?=E7=A7=92=E7=BA=A7=E6=9F=A5=E8=AF=A2=E6=8E=A5=E5=8F=A3-=20next?= =?UTF-8?q?=E5=AE=9E=E7=8E=B0=E9=A1=B5=E9=9D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dashboard/DashboardApplication.java | 60 +----- .../dashboard/common/base/BaseController.java | 4 +- .../common/config/MyExceptionHandler.java | 42 ---- .../{ExcelData.java => dto/ExcelDataDto.java} | 6 +- .../common/domain/dto/KeyCountDto.java | 39 ---- .../common/domain/vo/HitCountVo.java | 182 +++++++++++++++++ .../common/domain/vo/LineChartVo.java | 24 +++ .../common/domain/vo/RuleLineChartVo.java | 35 ---- .../dashboard/common/monitor/DataHandler.java | 8 +- .../dashboard/common/monitor/EtcdMonitor.java | 68 +++---- .../hotkey/dashboard/common/monitor/Test.java | 54 ----- .../dashboard/controller/KeyController.java | 16 +- .../dashboard/controller/RuleController.java | 10 + .../dashboard/controller/UserController.java | 9 - .../dashboard/mapper/KeyRuleMapper.java | 23 --- .../dashboard/mapper/KeyTimelyMapper.java | 2 + .../dashboard/mapper/ReceiveCountMapper.java | 31 --- .../dashboard/mapper/SummaryMapper.java | 6 + .../hotkey/dashboard/model/KeyRule.java | 136 ------------- .../hotkey/dashboard/model/KeyTimely.java | 71 +++++++ .../hotkey/dashboard/model/ReceiveCount.java | 98 --------- .../hotkey/dashboard/model/Summary.java | 130 ++++++++++-- .../hotkey/dashboard/service/KeyService.java | 2 - .../hotkey/dashboard/service/RuleService.java | 3 + .../service/impl/KeyServiceImpl.java | 38 +--- .../service/impl/RuleServiceImpl.java | 20 +- .../hotkey/dashboard/util/CommonUtil.java | 53 ++--- .../hotkey/dashboard/util/DateUtil.java | 5 + .../hotkey/dashboard/util/ExcelUtil.java | 4 +- .../dashboard/mapper/ChangeLogMapper.xml | 12 +- .../hotkey/dashboard/mapper/KeyRuleMapper.xml | 189 ------------------ .../dashboard/mapper/KeyTimelyMapper.xml | 49 +++++ .../dashboard/mapper/ReceiveCountMapper.xml | 50 ----- dashboard/src/main/resources/db.sql | 66 +----- 34 files changed, 561 insertions(+), 984 deletions(-) rename dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/{ExcelData.java => dto/ExcelDataDto.java} (82%) delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/dto/KeyCountDto.java create mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HitCountVo.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/Test.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRuleMapper.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ReceiveCountMapper.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRule.java delete mode 100644 dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/ReceiveCount.java delete mode 100644 dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyRuleMapper.xml delete mode 100644 dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ReceiveCountMapper.xml diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java index 1c295c2..074e45d 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/DashboardApplication.java @@ -1,15 +1,7 @@ package com.jd.platform.hotkey.dashboard; -import com.alibaba.fastjson.JSON; -import com.jd.platform.hotkey.common.configcenter.ConfigConstant; -import com.jd.platform.hotkey.common.configcenter.IConfigCenter; -import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import com.jd.platform.hotkey.common.tool.IpUtils; import com.jd.platform.hotkey.dashboard.mapper.KeyTimelyMapper; -import com.jd.platform.hotkey.dashboard.mapper.RulesMapper; -import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; -import com.jd.platform.hotkey.dashboard.util.CommonUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.boot.CommandLineRunner; @@ -17,10 +9,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.annotation.EnableScheduling; - import javax.annotation.Resource; -import java.util.HashMap; -import java.util.Map; @EnableAsync @@ -28,10 +17,8 @@ import java.util.Map; @SpringBootApplication public class DashboardApplication implements CommandLineRunner { - private Logger logger = LoggerFactory.getLogger(getClass()); - @Resource - private SummaryMapper summaryMapper; + @Resource private KeyTimelyMapper timelyMapper; @@ -41,52 +28,11 @@ public class DashboardApplication implements CommandLineRunner { }catch (Exception e){ e.printStackTrace(); } - } - @Override - public void run(String... args) { - /* Map map1 = new HashMap<>(); - String k1 = "cartsoa#**#rule1__#**#2020-10-23 21:11:21"; - String v1 = "50-200"; - String k2 = "cartsoa#**#rule2__#**#2020-10-23 21:11:22"; - String v2 = "30-100"; - map1.put(k1,v1); - map1.put(k2,v2); - for (String key : map1.keySet()) { - int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map1)); - } - Map map12 = new HashMap<>(); - String k12 = "cartsoa#**#rule1__#**#2020-10-23 21:11:21"; - String v12 = "10-100"; - String k22 = "cartsoa#**#rule2__#**#2020-10-23 21:11:23"; - String v22 = "70-300"; - String kk22 = "cartsoa#**#rule2__#**#2020-10-23 21:11:22"; - String vv22 = "87-500"; - String k33 = "cartsoa#**#rule3__#**#2020-10-23 21:11:22"; - String v33 = "72-300"; - map12.put(k12,v12); - map12.put(k22,v22); - map12.put(k33,v33); - map12.put(kk22,vv22); - for (String key : map12.keySet()) { - int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map12)); - } - - Map map122 = new HashMap<>(); - String k122 = "cartsoa#**#rule1__#**#2020-10-23 21:11:23"; - String v122 = "44-55"; - String k222 = "cartsoa#**#rule2__#**#2020-10-23 21:11:25"; - String v222 = "66-77"; - String k332 = "cartsoa#**#rule3__#**#2020-10-23 21:11:27"; - String v332 = "88-99"; - map122.put(k122,v122); - map122.put(k222,v222); - map122.put(k332,v332); - for (String key : map122.keySet()) { - int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map122)); - }*/ + @Override + public void run(String... args) { int row = timelyMapper.clear(); logger.info("clear db timely hotKey, effect row : {}",row); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java index 91555ff..7aa346f 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/base/BaseController.java @@ -41,7 +41,7 @@ public class BaseController { Claims claims = JwtTokenUtil.claims(authHeader.substring(2)); String role = claims.get("role",String.class); if(!role.equals(Constant.ADMIN)){ - if(!appName().equals(app)){ + if(!ownApp().equals(app)){ throw new BizException(ResultEnum.NO_PERMISSION); } } @@ -65,7 +65,7 @@ public class BaseController { } - public String appName(){ + public String ownApp(){ String authHeader = JwtTokenUtil.getAuthHeader(request); assert authHeader != null; Claims claims = JwtTokenUtil.claims(authHeader.substring(2)); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java index 99c2483..52559c9 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/config/MyExceptionHandler.java @@ -41,46 +41,4 @@ public class MyExceptionHandler { logger.info("未知异常：",e); return Result.error(ResultEnum.BIZ_ERROR); } - - - public static void main(String[] args) { - Random rd = new Random(); - - FileWriter fileWriter = null; - try { - fileWriter = new FileWriter("D:/tmp.txt");//创建文本文件 - // yyMMddHHmm - LocalDateTime time2 = DateUtil.strToLdt("2006140600", DateUtil.PATTERN_MINUS); - for (int i = 0; i < 160000; i++) { - int count = rd.nextInt(300); - int day = DateUtil.reviseTime(time2,i,1)/100/100; - int hour = DateUtil.reviseTime(time2,i,1)/100; - int ms = DateUtil.reviseTime(time2,i,1); - LocalDateTime ct = DateUtil.strToLdt(ms+"", DateUtil.PATTERN_MINUS); - String sql ="INSERT INTO `hk_statistics` VALUES ("+(i+55000)+", 'rule1', "+count+", 'app1', 'rule1', " + - ""+day+", "+hour+", "+ms+", 5, "+(i+55000)+", \""+ ct.toString().replace("T"," ")+"\" );"; - System.out.println(sql); - fileWriter.write(sql+"\r\n");//写入 \r\n换行 - } - fileWriter.flush(); - fileWriter.close(); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - - /* LocalDateTime time = DateUtil.strToLdt("20060200", DateUtil.PATTERN_HOUR); - for (int i = 0; i < 3000; i++) { - int count = rd.nextInt(500); - int day = DateUtil.reviseTime(time,i,2)/100; - int hour = DateUtil.reviseTime(time,i,2); - LocalDateTime ct = DateUtil.strToLdt(hour+"", DateUtil.PATTERN_HOUR); - String sql ="INSERT INTO `hk_statistics` VALUES ("+i+", 'rule1', "+count+", 'app1', 'rule1', " + - ""+day+", "+hour+", 0, 6, "+i+", \""+ ct.toString().replace("T"," ")+"\" );"; - System.out.println(sql); - }*/ - - - } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/ExcelData.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/dto/ExcelDataDto.java similarity index 82% rename from dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/ExcelData.java rename to dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/dto/ExcelDataDto.java index be933c6..8f6594e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/ExcelData.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/dto/ExcelDataDto.java @@ -1,11 +1,11 @@ -package com.jd.platform.hotkey.dashboard.common.domain; +package com.jd.platform.hotkey.dashboard.common.domain.dto; import java.util.List; /** * @author liyunfeng31 */ -public class ExcelData { +public class ExcelDataDto { /** * 文件名 */ @@ -19,7 +19,7 @@ public class ExcelData { */ private List> rows; - public ExcelData(String fileName, List head, List> rows) { + public ExcelDataDto(String fileName, List head, List> rows) { this.fileName = fileName; this.head = head; this.rows = rows; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/dto/KeyCountDto.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/dto/KeyCountDto.java deleted file mode 100644 index 976af9c..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/dto/KeyCountDto.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.jd.platform.hotkey.dashboard.common.domain.dto; - -import java.io.Serializable; - -/** - * @author liyunfeng31 - */ -public class KeyCountDto implements Serializable { - - private String k; - - private Integer count; - - private String app; - - public String getK() { - return k; - } - - public void setK(String k) { - this.k = k; - } - - public Integer getCount() { - return count; - } - - public void setCount(Integer count) { - this.count = count; - } - - public String getApp() { - return app; - } - - public void setApp(String app) { - this.app = app; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HitCountVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HitCountVo.java new file mode 100644 index 0000000..b0529dd --- /dev/null +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/HitCountVo.java @@ -0,0 +1,182 @@ +package com.jd.platform.hotkey.dashboard.common.domain.vo; + + +import java.io.Serializable; +import java.util.Date; + + +/** + * @author liyunfeng31 + */ +public class HitCountVo implements Serializable { + + private String rule; + + private String app; + + private Integer hitCount; + + private Integer totalCount; + + private Integer days; + + private Integer hours; + + private Integer minutes; + + private Integer seconds; + + private Date createTime; + + public String getRule() { + return rule; + } + + public void setRule(String rule) { + this.rule = rule; + } + + public String getApp() { + return app; + } + + public void setApp(String app) { + this.app = app; + } + + public Integer getHitCount() { + return hitCount; + } + + public void setHitCount(Integer hitCount) { + this.hitCount = hitCount; + } + + public Integer getTotalCount() { + return totalCount; + } + + public void setTotalCount(Integer totalCount) { + this.totalCount = totalCount; + } + + public Integer getDays() { + return days; + } + + public void setDays(Integer days) { + this.days = days; + } + + public Integer getHours() { + return hours; + } + + public void setHours(Integer hours) { + this.hours = hours; + } + + public Integer getMinutes() { + return minutes; + } + + public void setMinutes(Integer minutes) { + this.minutes = minutes; + } + + public Integer getSeconds() { + return seconds; + } + + public void setSeconds(Integer seconds) { + this.seconds = seconds; + } + + public Date getCreateTime() { + return createTime; + } + + public void setCreateTime(Date createTime) { + this.createTime = createTime; + } + + + public static HitCountVoBuilder aHitCountVo() { + return new HitCountVoBuilder(); + } + + + public static final class HitCountVoBuilder { + private String rule; + private String app; + private Integer hitCount; + private Integer totalCount; + private Integer days; + private Integer hours; + private Integer minutes; + private Integer seconds; + private Date createTime; + + private HitCountVoBuilder() { + } + + public HitCountVoBuilder withRule(String rule) { + this.rule = rule; + return this; + } + + public HitCountVoBuilder withApp(String app) { + this.app = app; + return this; + } + + public HitCountVoBuilder withHitCount(Integer hitCount) { + this.hitCount = hitCount; + return this; + } + + public HitCountVoBuilder withTotalCount(Integer totalCount) { + this.totalCount = totalCount; + return this; + } + + public HitCountVoBuilder withDays(Integer days) { + this.days = days; + return this; + } + + public HitCountVoBuilder withHours(Integer hours) { + this.hours = hours; + return this; + } + + public HitCountVoBuilder withMinutes(Integer minutes) { + this.minutes = minutes; + return this; + } + + public HitCountVoBuilder withSeconds(Integer seconds) { + this.seconds = seconds; + return this; + } + + public HitCountVoBuilder withCreateTime(Date createTime) { + this.createTime = createTime; + return this; + } + + public HitCountVo build() { + HitCountVo hitCountVo = new HitCountVo(); + hitCountVo.setRule(rule); + hitCountVo.setApp(app); + hitCountVo.setHitCount(hitCount); + hitCountVo.setTotalCount(totalCount); + hitCountVo.setDays(days); + hitCountVo.setHours(hours); + hitCountVo.setMinutes(minutes); + hitCountVo.setSeconds(seconds); + hitCountVo.setCreateTime(createTime); + return hitCountVo; + } + } +} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java index 2ae730a..273ce59 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/LineChartVo.java @@ -9,6 +9,8 @@ import java.util.Set; */ public class LineChartVo { + private Set legend; + private Set xAxis; private Map> series; @@ -18,11 +20,33 @@ public class LineChartVo { this.series = series; } + public LineChartVo(Set legend, Set xAxis, Map> series) { + this.legend = legend; + this.xAxis = xAxis; + this.series = series; + } + public Set getxAxis() { return xAxis; } + public void setxAxis(Set xAxis) { + this.xAxis = xAxis; + } + public Map> getSeries() { return series; } + + public void setSeries(Map> series) { + this.series = series; + } + + public Set getLegend() { + return legend; + } + + public void setLegend(Set legend) { + this.legend = legend; + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java deleted file mode 100644 index 31ba530..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/domain/vo/RuleLineChartVo.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.jd.platform.hotkey.dashboard.common.domain.vo; - -/** - * @author liyunfeng31 - */ -public class RuleLineChartVo { - - private String rule; - - private Integer count; - - public RuleLineChartVo() { - } - - public RuleLineChartVo(String rule, Integer count) { - this.rule = rule; - this.count = count; - } - - public String getRule() { - return rule; - } - - public void setRule(String rule) { - this.rule = rule; - } - - public Integer getCount() { - return count; - } - - public void setCount(Integer count) { - this.count = count; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java index c98c427..4c9ba3e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/DataHandler.java @@ -72,7 +72,7 @@ public class DataHandler { keyTimelyMapper.deleteByKeyAndApp(keyTimely.getKey(), keyTimely.getAppName()); } else { try { - keyTimelyMapper.insertSelective(keyTimely); + keyTimelyMapper.saveOrUpdate(keyTimely); } catch (Exception e) { log.info("insert timely error"); } @@ -109,16 +109,14 @@ public class DataHandler { if (eventType.equals(Event.EventType.PUT)) { //手工添加的是时间戳13位，worker传过来的是uuid String source = value.length() == 13 ? Constant.HAND : Constant.SYSTEM; - timelyKeyRecordTwoTuple.setFirst(new KeyTimely(key, value, appName, ttl, uuid, date)); - - // 这是一个骚操作 在线上不方便新增rule字段的时候 临时用下val + timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).val(value).appName(appName).duration(ttl).uuid(appKey).createTime(date).build()); String rule = RuleUtil.rule(appKey); KeyRecord keyRecord = new KeyRecord(key, rule, appName, ttl, source, type, uuid, date); keyRecord.setRule(rule); timelyKeyRecordTwoTuple.setSecond(keyRecord); return timelyKeyRecordTwoTuple; } else if (eventType.equals(Event.EventType.DELETE)) { - timelyKeyRecordTwoTuple.setFirst(new KeyTimely(key, null, appName, 0L, null, null)); + timelyKeyRecordTwoTuple.setFirst(KeyTimely.aKeyTimely().key(key).appName(appName).build()); return timelyKeyRecordTwoTuple; } return timelyKeyRecordTwoTuple; diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java index 27369d5..92fa472 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/EtcdMonitor.java @@ -8,12 +8,9 @@ import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.common.rule.KeyRule; import com.jd.platform.hotkey.common.tool.FastJsonUtils; -import com.jd.platform.hotkey.common.tool.IpUtils; import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.EventWrapper; -import com.jd.platform.hotkey.dashboard.mapper.ReceiveCountMapper; import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; -import com.jd.platform.hotkey.dashboard.model.ReceiveCount; import com.jd.platform.hotkey.dashboard.model.Worker; import com.jd.platform.hotkey.dashboard.service.WorkerService; import com.jd.platform.hotkey.dashboard.util.CommonUtil; @@ -22,10 +19,14 @@ import io.grpc.StatusRuntimeException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; import javax.annotation.PostConstruct; import javax.annotation.Resource; -import java.util.*; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -53,8 +54,6 @@ public class EtcdMonitor { @Resource private DataHandler dataHandler; - @Resource - private ReceiveCountMapper receiveCountMapper; public static final ExecutorService threadPoolExecutor = Executors.newCachedThreadPool(); @@ -142,19 +141,20 @@ public class EtcdMonitor { */ private void watchRule() { threadPoolExecutor.submit(() -> { - try { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.rulePath); - //如果有新事件，即rule的变更，就重新拉取所有的信息 - while (watchIterator.hasNext()) { - //这句必须写，next会让他卡住，除非真的有新rule变更 - Event event = event(watchIterator); + KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.rulePath); + //如果有新事件，即rule的变更，就重新拉取所有的信息 + while (watchIterator.hasNext()) { + //这句必须写，next会让他卡住，除非真的有新rule变更 + Event event = event(watchIterator); + + try { log.info("---------watch rule change---------"); //全量拉取rule信息 fetchRuleFromEtcd(); + } catch (Exception e) { + log.error("watch rule err"); } - } catch (Exception e) { - log.error("watch rule err"); } }); @@ -216,50 +216,38 @@ public class EtcdMonitor { }); } - private void watchReceiveKeyCount() { - threadPoolExecutor.submit(() -> { - KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.totalReceiveKeyCount); - while (watchIterator.hasNext()) { - Event event = event(watchIterator); - KeyValue kv = event.getKv(); - Event.EventType eventType = event.getType(); - String k = kv.getKey().toStringUtf8(); - String v = kv.getValue().toStringUtf8(); - long version = kv.getModRevision(); - String uuid = k + Constant.JOIN + version; - if (eventType.equals(Event.EventType.PUT)) { - receiveCountMapper.insert(new ReceiveCount(k, Long.parseLong(v), uuid)); - } else if (eventType.equals(Event.EventType.DELETE)) { - receiveCountMapper.insert(new ReceiveCount(k, 0L, uuid)); - } - } - }); - } + /** * 监听热key访问次数和总访问次数 */ private void watchHitCount() { threadPoolExecutor.submit(() -> { - //key：ConfigConstant.keyHitCountPath + appName + "/" + IpUtils.getIp() + "-" + System.currentTimeMillis() - //value：FastJsonUtils.convertObjectToJSON(map) - //map的key是 appName + #**# + pin__#**#2020-10-23 21:11:22 - //map的value是"67-1937" 前面是热key访问量，后面是总访问量 KvClient.WatchIterator watchIterator = configCenter.watchPrefix(ConfigConstant.keyHitCountPath); while (watchIterator.hasNext()) { Event event = event(watchIterator); KeyValue kv = event.getKv(); + Event.EventType eventType = event.getType(); + if (Event.EventType.DELETE.equals(eventType)) { + continue; + } String k = kv.getKey().toStringUtf8(); String v = kv.getValue().toStringUtf8(); - Map map = FastJsonUtils.stringToCollect(v); - for (String key : map.keySet()) { - int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map)); + try { + Map map = FastJsonUtils.stringToCollect(v); + for (String key : map.keySet()) { + int row = summaryMapper.saveOrUpdate(CommonUtil.buildSummary(key, map)); + } + } catch (Exception e) { + e.printStackTrace(); } + } }); } + private Event event(KvClient.WatchIterator watchIterator) { return watchIterator.next().getEvents().get(0); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/Test.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/Test.java deleted file mode 100644 index 017f42c..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/common/monitor/Test.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.jd.platform.hotkey.dashboard.common.monitor; - -import java.util.ArrayList; -import java.util.List; -import java.util.Objects; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; -import java.util.stream.Collectors; - -/** - * @author wuweifeng - * @version 1.0 - * @date 2020-05-18 - */ -public class Test { - public static void main(String[] args) throws InterruptedException { - ConcurrentLinkedQueue queue = new ConcurrentLinkedQueue<>(); - for (int i = 0; i < 19000; i++) { - queue.offer(i + ""); - } - Executor executor = Executors.newFixedThreadPool(4); - while (true) { - List keyRecords = new ArrayList<>(10000); - for (int i = 0; i < 10000; i++) { - if (!queue.isEmpty()) { - keyRecords.add(queue.poll()); - } else { - keyRecords.add(null); - } - } - - for (int i = 0; i < 10; i++) { - List tempRecords = keyRecords.subList(1000 * i, 1000 * (i + 1)); - executor.execute(() -> batch(tempRecords)); - } - - Thread.sleep(1000); - } - - } - - private static void batch(List strings) { - List records = strings.stream().filter(Objects::nonNull).collect(Collectors.toList()); - if (records.size() > 0) { - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - System.out.println(records.size()); - } - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java index bac86cc..0b287a8 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/KeyController.java @@ -4,7 +4,7 @@ import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; import com.jd.platform.hotkey.dashboard.common.base.BaseController; import com.jd.platform.hotkey.dashboard.common.domain.Constant; -import com.jd.platform.hotkey.dashboard.common.domain.ExcelData; +import com.jd.platform.hotkey.dashboard.common.domain.dto.ExcelDataDto; import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.Result; import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; @@ -44,8 +44,7 @@ public class KeyController extends BaseController { @PostMapping("/ruleLineChart") @ResponseBody public HotKeyLineChartVo ruleLineChart(SearchReq req){ - String app = appName(); - return keyService.ruleLineChart(req,app); + return keyService.ruleLineChart(req,ownApp()); } @@ -56,15 +55,6 @@ public class KeyController extends BaseController { } - //@PostMapping("/qps") - @GetMapping("/qps") - @ResponseBody - public HotKeyLineChartVo qpsLineChart(ChartReq ChartReq) { - return keyService.getQpsLineChart(ChartReq); - } - - - @GetMapping("/view") public String view(ModelMap modelMap){ modelMap.put("title", Constant.KEY_RECORD_VIEW); @@ -163,7 +153,7 @@ public class KeyController extends BaseController { req.setKey(key); List records = keyService.listMaxHot(req); List> rows = transform(records); - ExcelData data = new ExcelData("hotKey.xlsx", Constant.HEAD,rows); + ExcelDataDto data = new ExcelDataDto("hotKey.xlsx", Constant.HEAD,rows); ExcelUtil.exportExcel(resp,data); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java index 221581a..0946e7c 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/RuleController.java @@ -10,6 +10,8 @@ import com.jd.platform.hotkey.dashboard.common.domain.Constant; import com.jd.platform.hotkey.dashboard.common.domain.Page; import com.jd.platform.hotkey.dashboard.common.domain.Result; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; import com.jd.platform.hotkey.dashboard.common.eunm.ResultEnum; import com.jd.platform.hotkey.dashboard.common.ex.BizException; import com.jd.platform.hotkey.dashboard.model.Rule; @@ -107,6 +109,14 @@ public class RuleController extends BaseController { } + @PostMapping("/listRuleHitCount") + @ResponseBody + public Page pageRuleHitCount(PageReq page, SearchReq req){ + PageInfo info = ruleService.pageRuleHitCount(page, req, ownApp()); + return new Page<>(info.getPageNum(),(int)info.getTotal(),info.getList()); + } + + /** * 校验是否合法 * @param rules rules diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java index 1388c6a..6705366 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/controller/UserController.java @@ -184,15 +184,6 @@ public class UserController extends BaseController { @PostMapping("getUserName") @ResponseBody public String getUserName(HttpServletRequest request, HttpServletResponse response){ - /*Cookie[] cookies = request.getCookies(); - String userName = ""; - for(Cookie cookie : cookies){ - if("erp".equals(cookie.getName())){ - userName = cookie.getValue(); - break; - } - } - return userName;*/ return userName(); } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRuleMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRuleMapper.java deleted file mode 100644 index e9f4169..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyRuleMapper.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.jd.platform.hotkey.dashboard.mapper; - -import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; -import com.jd.platform.hotkey.dashboard.model.KeyRule; -import org.apache.ibatis.annotations.Mapper; -import org.apache.ibatis.annotations.Options; - -import java.util.List; - -@Mapper -public interface KeyRuleMapper { - - @Options(useGeneratedKeys = true) - int insertSelective(KeyRule record); - - KeyRule selectByKey(String key); - - int updateByKey(KeyRule record); - - List listRule(SearchReq param); - - void batchInsert(List rules); -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java index 826352a..3423da3 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.java @@ -26,4 +26,6 @@ public interface KeyTimelyMapper { int batchInsert(List list); void batchDeleted(List deleteList); + + void saveOrUpdate(KeyTimely keyTimely); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ReceiveCountMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ReceiveCountMapper.java deleted file mode 100644 index 6593ce0..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/ReceiveCountMapper.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.jd.platform.hotkey.dashboard.mapper; - - -import com.jd.platform.hotkey.dashboard.common.domain.req.ChartReq; -import com.jd.platform.hotkey.dashboard.model.ReceiveCount; -import org.apache.ibatis.annotations.Mapper; - -import java.util.List; - - -/** - * @author liyunfeng31 - */ -@Mapper -public interface ReceiveCountMapper { - - /** - * 插入最新记录 - * @param record record - * @return row - */ - int insert(ReceiveCount record); - - /** - * 查询统计 - * @param chartReq req - * @return list - */ - List list(ChartReq chartReq); - -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java index 90aaff6..4685e76 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/mapper/SummaryMapper.java @@ -1,8 +1,13 @@ package com.jd.platform.hotkey.dashboard.mapper; +import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; +import com.jd.platform.hotkey.dashboard.model.Statistics; import com.jd.platform.hotkey.dashboard.model.Summary; import org.apache.ibatis.annotations.Mapper; +import java.util.List; + /** * @author liyunfeng31 */ @@ -16,4 +21,5 @@ public interface SummaryMapper { */ int saveOrUpdate(Summary records); + List listRuleHitCount(SearchReq req); } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRule.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRule.java deleted file mode 100644 index 5231a38..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyRule.java +++ /dev/null @@ -1,136 +0,0 @@ -package com.jd.platform.hotkey.dashboard.model; - -import java.util.Date; - -public class KeyRule { - private Integer id; - - /** - * key的前缀，也可以完全和key相同。为"*"时代表通配符 - */ - private String key; - /** - * 是否是前缀，true是前缀 - */ - private Boolean prefix; - /** - * 间隔时间（秒） - */ - private Integer interval; - /** - * 累计数量 - */ - private Integer threshold; - /** - * 变热key后，本地、etcd缓存它多久。单位（秒），默认60 - */ - private Integer duration; - - private String appName; - - private Integer state; - - private String updateUser; - - private Date updateTime; - - private Integer version; - - public KeyRule() { - } - - public KeyRule(String key, Integer state,String updateUser) { - this.key = key; - this.state = state; - this.updateUser = updateUser; - } - - public Integer getId() { - return id; - } - - public void setId(Integer id) { - this.id = id; - } - - public String getKey() { - return key; - } - - public void setKey(String key) { - this.key = key; - } - - public Boolean getPrefix() { - return prefix; - } - - public void setPrefix(Boolean prefix) { - this.prefix = prefix; - } - - public Integer getInterval() { - return interval; - } - - public void setInterval(Integer interval) { - this.interval = interval; - } - - public Integer getThreshold() { - return threshold; - } - - public void setThreshold(Integer threshold) { - this.threshold = threshold; - } - - public Integer getDuration() { - return duration; - } - - public void setDuration(Integer duration) { - this.duration = duration; - } - - public String getAppName() { - return appName; - } - - public void setAppName(String appName) { - this.appName = appName == null ? null : appName.trim(); - } - - public Integer getState() { - return state; - } - - public void setState(Integer state) { - this.state = state; - } - - public String getUpdateUser() { - return updateUser; - } - - public void setUpdateUser(String updateUser) { - this.updateUser = updateUser == null ? null : updateUser.trim(); - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - public Integer getVersion() { - return version; - } - - public void setVersion(Integer version) { - this.version = version; - } - -} \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java index 86389c3..6efd7c5 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/KeyTimely.java @@ -121,4 +121,75 @@ public class KeyTimely implements Serializable { public void setUpdater(String updater) { this.updater = updater; } + + public static Builder aKeyTimely() { + return new Builder(); + } + + public static final class Builder { + private String key; + private String appName; + private String val; + private Long duration; + private Date createTime; + private String uuid; + private transient String ruleDesc; + private String updater; + + private Builder() { + } + + public Builder key(String key) { + this.key = key; + return this; + } + + public Builder appName(String appName) { + this.appName = appName; + return this; + } + + public Builder val(String val) { + this.val = val; + return this; + } + + public Builder duration(Long duration) { + this.duration = duration; + return this; + } + + public Builder createTime(Date createTime) { + this.createTime = createTime; + return this; + } + + public Builder uuid(String uuid) { + this.uuid = uuid; + return this; + } + + public Builder ruleDesc(String ruleDesc) { + this.ruleDesc = ruleDesc; + return this; + } + + public Builder updater(String updater) { + this.updater = updater; + return this; + } + + public KeyTimely build() { + KeyTimely keyTimely = new KeyTimely(); + keyTimely.setKey(key); + keyTimely.setAppName(appName); + keyTimely.setVal(val); + keyTimely.setDuration(duration); + keyTimely.setCreateTime(createTime); + keyTimely.setUuid(uuid); + keyTimely.setRuleDesc(ruleDesc); + keyTimely.setUpdater(updater); + return keyTimely; + } + } } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/ReceiveCount.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/ReceiveCount.java deleted file mode 100644 index 7376eaa..0000000 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/ReceiveCount.java +++ /dev/null @@ -1,98 +0,0 @@ -package com.jd.platform.hotkey.dashboard.model; - -import com.jd.platform.hotkey.dashboard.util.DateUtil; - -import java.io.Serializable; -import java.time.LocalDateTime; -import java.util.Date; - -/** - * @author liyunfeng31 - */ -public class ReceiveCount implements Serializable { - - private Long id; - - private String workerName; - - private Long receiveCount; - - private String uuid; - - private Date updateTime; - - private Integer hour; - - private Integer minutes; - - - public ReceiveCount() { - } - - public ReceiveCount(String workerName, Long receiveCount,String uuid) { - this.workerName = workerName; - this.receiveCount = receiveCount; - this.uuid = uuid; - this.updateTime = new Date(); - LocalDateTime now = LocalDateTime.now(); - this.hour = DateUtil.nowHour(now); - this.minutes = DateUtil.nowMinus(now); - } - - - public Long getId() { - return id; - } - - public void setId(Long id) { - this.id = id; - } - - public String getWorkerName() { - return workerName; - } - - public void setWorkerName(String workerName) { - this.workerName = workerName; - } - - public Long getReceiveCount() { - return receiveCount; - } - - public void setReceiveCount(Long receiveCount) { - this.receiveCount = receiveCount; - } - - public String getUuid() { - return uuid; - } - - public void setUuid(String uuid) { - this.uuid = uuid; - } - - public Date getUpdateTime() { - return updateTime; - } - - public void setUpdateTime(Date updateTime) { - this.updateTime = updateTime; - } - - public Integer getHour() { - return hour; - } - - public void setHour(Integer hour) { - this.hour = hour; - } - - public Integer getMinutes() { - return minutes; - } - - public void setMinutes(Integer minutes) { - this.minutes = minutes; - } -} diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/Summary.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/Summary.java index a8b85dc..ddf90a6 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/Summary.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/model/Summary.java @@ -41,22 +41,6 @@ public class Summary implements Serializable { public Summary() { } - public Summary(String indexName, String rule, String app, Integer totalCount, Integer hitCount, Integer days, Integer hours, Integer minutes, Integer seconds, Integer bizType, String uuid) { - this.indexName = indexName; - this.rule = rule; - this.app = app; - this.val1 = totalCount; - this.val2 = hitCount; - this.val3 = BigDecimal.ZERO; - this.days = days; - this.hours = hours; - this.minutes = minutes; - this.seconds = seconds; - this.bizType = bizType; - this.uuid = uuid; - this.createTime = new Date(); - } - public Integer getId() { return id; } @@ -168,4 +152,118 @@ public class Summary implements Serializable { public void setCreateTime(Date createTime) { this.createTime = createTime; } + + + public static SummaryBuilder aSummary() { + return new SummaryBuilder(); + } + + public static final class SummaryBuilder { + private Integer id; + private String indexName; + private String rule; + private String app; + private Integer val1; + private Integer val2; + private BigDecimal val3; + private Integer days; + private Integer hours; + private Integer minutes; + private Integer seconds; + private Integer bizType; + private String uuid; + private Date createTime; + + private SummaryBuilder() { + } + + public SummaryBuilder id(Integer id) { + this.id = id; + return this; + } + + public SummaryBuilder indexName(String indexName) { + this.indexName = indexName; + return this; + } + + public SummaryBuilder rule(String rule) { + this.rule = rule; + return this; + } + + public SummaryBuilder app(String app) { + this.app = app; + return this; + } + + public SummaryBuilder val1(Integer val1) { + this.val1 = val1; + return this; + } + + public SummaryBuilder val2(Integer val2) { + this.val2 = val2; + return this; + } + + public SummaryBuilder val3(BigDecimal val3) { + this.val3 = val3; + return this; + } + + public SummaryBuilder days(Integer days) { + this.days = days; + return this; + } + + public SummaryBuilder hours(Integer hours) { + this.hours = hours; + return this; + } + + public SummaryBuilder minutes(Integer minutes) { + this.minutes = minutes; + return this; + } + + public SummaryBuilder seconds(Integer seconds) { + this.seconds = seconds; + return this; + } + + public SummaryBuilder bizType(Integer bizType) { + this.bizType = bizType; + return this; + } + + public SummaryBuilder uuid(String uuid) { + this.uuid = uuid; + return this; + } + + public SummaryBuilder createTime(Date createTime) { + this.createTime = createTime; + return this; + } + + public Summary build() { + Summary summary = new Summary(); + summary.setId(id); + summary.setIndexName(indexName); + summary.setRule(rule); + summary.setApp(app); + summary.setVal1(val1); + summary.setVal2(val2); + summary.setVal3(val3); + summary.setDays(days); + summary.setHours(hours); + summary.setMinutes(minutes); + summary.setSeconds(seconds); + summary.setBizType(bizType); + summary.setUuid(uuid); + summary.setCreateTime(createTime); + return summary; + } + } } \ No newline at end of file diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java index cc93ea7..3dfcf18 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/KeyService.java @@ -41,7 +41,5 @@ public interface KeyService { HotKeyLineChartVo getLineChart(ChartReq chartReq); - HotKeyLineChartVo getQpsLineChart(ChartReq chartReq); - HotKeyLineChartVo ruleLineChart(SearchReq req, String app); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java index d07a728..5580df1 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/RuleService.java @@ -3,6 +3,7 @@ package com.jd.platform.hotkey.dashboard.service; import com.github.pagehelper.PageInfo; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; import com.jd.platform.hotkey.dashboard.model.Rules; import java.util.List; @@ -30,4 +31,6 @@ public interface RuleService { int save(Rules rules); List listRules(String app); + + PageInfo pageRuleHitCount(PageReq pageReq, SearchReq req, String ownApp); } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java index 6a03a29..5b6bfa2 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/KeyServiceImpl.java @@ -32,6 +32,7 @@ import java.util.stream.Collectors; /** * @ProjectName: hotkey * @ClassName: KeyServiceImpl + * @Description: TODO(一句话描述该类的功能) * @Author: liyunfeng31 * @Date: 2020/4/17 17:53 */ @@ -45,8 +46,6 @@ public class KeyServiceImpl implements KeyService { @Resource private KeyTimelyMapper keyTimelyMapper; @Resource - private ReceiveCountMapper countMapper; - @Resource private StatisticsMapper statisticsMapper; @Resource private RuleService ruleService; @@ -147,41 +146,6 @@ public class KeyServiceImpl implements KeyService { } - @Override - public HotKeyLineChartVo getQpsLineChart(ChartReq chartReq) { - if(chartReq.getStartTime() == null || chartReq.getEndTime() == null){ - /* chartReq.setStartTime(DateUtil.preTime()); - chartReq.setEndTime(new Date());*/ - } - List countList = countMapper.list(chartReq); - Map map = new HashMap<>(10); - Set minutes = new HashSet<>(); - List> workerList = new ArrayList<>(); - countList.stream().collect(Collectors.groupingBy(ReceiveCount::getWorkerName,Collectors.toList())) - .forEach((name,data)-> workerList.add(data)); - for (List cts : workerList) { - int size = cts.size(); - for (int i = 0; i < size; i++) { - ReceiveCount dto = cts.get(i); - String k = dto.getWorkerName(); - Long v = dto.getReceiveCount(); - Integer ms = dto.getMinutes(); - minutes.add(ms.toString()); - if(map.get(k) == null){ - int [] data = new int[size]; - data[i] = v.intValue(); - map.put(k, data); - }else{ - int [] data = map.get(k); - data[i] = v.intValue(); - map.put(k, data); - } - } - } - return new HotKeyLineChartVo(new ArrayList<>(minutes),map); - } - - @Override public PageInfo pageKeyRecord(PageReq page, SearchReq param) { PageHelper.startPage(page.getPageNum(), page.getPageSize()); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java index 606ad63..9dd493e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/RuleServiceImpl.java @@ -2,17 +2,19 @@ package com.jd.platform.hotkey.dashboard.service.impl; import cn.hutool.core.lang.UUID; import com.alibaba.fastjson.JSON; +import com.github.pagehelper.PageHelper; import com.github.pagehelper.PageInfo; import com.github.pagehelper.util.StringUtil; import com.ibm.etcd.api.KeyValue; import com.jd.platform.hotkey.common.configcenter.ConfigConstant; import com.jd.platform.hotkey.common.configcenter.IConfigCenter; import com.jd.platform.hotkey.dashboard.common.domain.req.PageReq; +import com.jd.platform.hotkey.dashboard.common.domain.req.SearchReq; +import com.jd.platform.hotkey.dashboard.common.domain.vo.HitCountVo; import com.jd.platform.hotkey.dashboard.mapper.ChangeLogMapper; import com.jd.platform.hotkey.dashboard.mapper.RulesMapper; -import com.jd.platform.hotkey.dashboard.model.ChangeLog; -import com.jd.platform.hotkey.dashboard.model.Rule; -import com.jd.platform.hotkey.dashboard.model.Rules; +import com.jd.platform.hotkey.dashboard.mapper.SummaryMapper; +import com.jd.platform.hotkey.dashboard.model.*; import com.jd.platform.hotkey.dashboard.service.RuleService; import org.springframework.stereotype.Service; import org.springframework.transaction.annotation.Transactional; @@ -43,6 +45,8 @@ public class RuleServiceImpl implements RuleService { @Resource private ChangeLogMapper logMapper; + @Resource + private SummaryMapper summaryMapper; @Override @@ -142,4 +146,14 @@ public class RuleServiceImpl implements RuleService { } return rules; } + + @Override + public PageInfo pageRuleHitCount(PageReq pageReq, SearchReq req, String ownApp) { + PageHelper.startPage(pageReq.getPageNum(),pageReq.getPageSize()); + List hitCountVos = summaryMapper.listRuleHitCount(req); + return new PageInfo<>(hitCountVos); + } + + + } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java index e91c5ac..85a1e75 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/CommonUtil.java @@ -204,28 +204,33 @@ public class CommonUtil { } - /** - * build存入对象 - * @param key key是 appName + #**# + pin__#**#2020-10-23 21:11:22 - * @param map value是"67-1937" 前面是热key访问量，后面是总访问量 - * @return Summary - */ - public static Summary buildSummary(String key, Map map){ - String[] args = key.split(com.jd.platform.hotkey.common.tool.Constant.BAK_DELIMITER); - String app = args[0]; - String rule = args[1]; - String hitTime = args[2]; - Date time = DateUtil.strToDate(hitTime); - assert time != null; - LocalDateTime ldt = DateUtil.dateToLdt(time); - int day = DateUtil.nowDay(ldt); - int hour = DateUtil.nowHour(ldt); - int minus = DateUtil.nowMinus(ldt); - long seconds = time.getTime()/1000; - String[] counts = map.get(key).split("-"); - int hitCount = Integer.parseInt(counts[0]); - int totalCount = Integer.parseInt(counts[1]); - String uuid =app+rule+hitTime; - return new Summary(rule,rule,app,totalCount,hitCount,day,hour,minus,(int)seconds,1,uuid); - } + /** + * build存入对象 + * + * @param key key是 appName + #**# + pin__#**#2020-10-23 21:11:22 + * @param map value是"67-1937" 前面是热key访问量，后面是总访问量 + * @return Summary + */ + public static Summary buildSummary(String key, Map map) { + String[] args = key.split(com.jd.platform.hotkey.common.tool.Constant.BAK_DELIMITER); + String app = args[0]; + String rule = args[1]; + String hitTime = args[2]; + Date time = DateUtil.strToDate(hitTime); + assert time != null; + LocalDateTime ldt = DateUtil.dateToLdt(time); + int day = DateUtil.nowDay(ldt); + int hour = DateUtil.nowHour(ldt); + int minus = DateUtil.nowMinus(ldt); + long seconds = time.getTime() / 1000; + String[] counts = map.get(key).split("-"); + int hitCount = Integer.parseInt(counts[0]); + int totalCount = Integer.parseInt(counts[1]); + String uuid = app + "-" + rule + hitTime; + + return Summary.aSummary().indexName(rule).rule(rule).app(app) + .val1(totalCount).val2(hitCount) + .days(day).hours(hour).minutes(minus).seconds((int) seconds) + .bizType(1).uuid(uuid).build(); + } } diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java index c049d78..8232567 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/DateUtil.java @@ -12,6 +12,7 @@ import java.util.Date; */ public class DateUtil { + public static final String PATTERN_SECONDS="yyMMddHHmmss"; public static final String PATTERN_MINUS="yyMMddHHmm"; @@ -19,6 +20,8 @@ public class DateUtil { public static final String PATTERN_DAY="yyMMdd"; + private static final DateTimeFormatter FORMAT_SECONDS = DateTimeFormatter.ofPattern(PATTERN_SECONDS); + private static final DateTimeFormatter FORMAT_MINUS = DateTimeFormatter.ofPattern(PATTERN_MINUS); private static final DateTimeFormatter FORMAT_HOUR = DateTimeFormatter.ofPattern(PATTERN_HOUR); @@ -44,6 +47,8 @@ public class DateUtil { public static int reviseTime(LocalDateTime time, int diff, int type){ switch (type){ + case 0: + return Integer.parseInt(FORMAT_SECONDS.format(time.plusSeconds(diff))); case 1: return Integer.parseInt(FORMAT_MINUS.format(time.plusMinutes(diff))); case 2: diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/ExcelUtil.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/ExcelUtil.java index 1c6370a..7641f0e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/ExcelUtil.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/util/ExcelUtil.java @@ -4,7 +4,7 @@ import javax.servlet.http.HttpServletResponse; import java.io.OutputStream; import java.util.List; -import com.jd.platform.hotkey.dashboard.common.domain.ExcelData; +import com.jd.platform.hotkey.dashboard.common.domain.dto.ExcelDataDto; import org.apache.poi.hssf.usermodel.HSSFDataFormat; import org.apache.poi.xssf.usermodel.*; import org.slf4j.Logger; @@ -15,7 +15,7 @@ public class ExcelUtil { private static Logger log = LoggerFactory.getLogger(ExcelUtil.class); - public static void exportExcel(HttpServletResponse response, ExcelData data) { + public static void exportExcel(HttpServletResponse response, ExcelDataDto data) { log.info("导出解析开始，fileName:{}",data.getFileName()); try { //实例化HSSFWorkbook diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml index fb2a15f..90b3b5d 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ChangeLogMapper.xml @@ -3,7 +3,7 @@ - + @@ -13,7 +13,7 @@ - id, biz_key, biz_type, from_str, to_str, app_name,update_user, create_time,uuid + id, biz_id, biz_type, from_str, to_str, app_name,update_user, create_time,uuid - select - - from hk_key_rule - where id = #{id,jdbcType=INTEGER} - - - - - - insert ignore into hk_key_rule - - - id, - - - key_name, - - - prefix, - - - intervals, - - - threshold, - - - duration, - - - app_name, - - - state, - - - update_user, - - - update_time, - - - version, - - - - - #{id,jdbcType=INTEGER}, - - - #{key,jdbcType=VARCHAR}, - - - #{prefix,jdbcType=INTEGER}, - - - #{interval,jdbcType=INTEGER}, - - - #{threshold,jdbcType=INTEGER}, - - - #{duration,jdbcType=INTEGER}, - - - #{appName,jdbcType=VARCHAR}, - - - #{state,jdbcType=INTEGER}, - - - #{updateUser,jdbcType=VARCHAR}, - - - #{updateTime,jdbcType=TIMESTAMP}, - - - #{version,jdbcType=INTEGER}, - - - - - - update hk_key_rule - - - key_name = #{key,jdbcType=VARCHAR}, - - - prefix = #{prefix,jdbcType=INTEGER}, - - - intervals = #{interval,jdbcType=INTEGER}, - - - threshold = #{threshold,jdbcType=INTEGER}, - - - duration = #{duration,jdbcType=INTEGER}, - - - app_name = #{appName,jdbcType=VARCHAR}, - - - state = #{state,jdbcType=INTEGER}, - - - update_user = #{updateUser,jdbcType=VARCHAR}, - - - update_time = #{updateTime,jdbcType=TIMESTAMP}, - - - version = #{version,jdbcType=INTEGER}, - - - where key_name = #{key} - - - - - - - - INSERT ignore INTO - hk_key_rule( - key_name, - prefix, - intervals, - threshold, - duration, - app_name, - state, - update_user, - update_time) - VALUES - - (#{k.key}, - #{k.prefix}, - #{k.interval}, - #{k.threshold}, - #{k.duration}, - #{k.appName}, - #{k.state}, - #{k.updateUser}, - #{k.updateTime}) - - - - \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml index 79e25e8..04704cf 100644 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml +++ b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/KeyTimelyMapper.xml @@ -155,4 +155,53 @@ order by id desc + + + + + insert into hk_key_timely + + + key_name, + + + val, + + + uuid, + + + app_name, + + + duration, + + + create_time, + + + + + #{key,jdbcType=VARCHAR}, + + + #{val,jdbcType=VARCHAR}, + + + #{uuid,jdbcType=VARCHAR}, + + + #{appName,jdbcType=VARCHAR}, + + + #{duration,jdbcType=BIGINT}, + + + #{createTime,jdbcType=TIMESTAMP}, + + + on duplicate key update + create_time = now() + + \ No newline at end of file diff --git a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ReceiveCountMapper.xml b/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ReceiveCountMapper.xml deleted file mode 100644 index 1bc638b..0000000 --- a/dashboard/src/main/resources/com/jd/platform/hotkey/dashboard/mapper/ReceiveCountMapper.xml +++ /dev/null @@ -1,50 +0,0 @@ - - - - - - - - - - - - - - id, receive_count, worker_name, uuid,create_time,hours,minutes - - - - delete from hk_receive_count - where id = #{id,jdbcType=BIGINT} - - - insert into hk_receive_count (id, worker_name, receive_count, - hours, minutes, uuid, create_time) - values (#{id,jdbcType=BIGINT}, #{workerName,jdbcType=VARCHAR}, #{receiveCount,jdbcType=BIGINT}, - #{hours,jdbcType=BIGINT}, #{minutes,jdbcType=BIGINT}, #{uuid,jdbcType=VARCHAR}, - #{createTime,jdbcType=TIMESTAMP}) - - - - - - - - - \ No newline at end of file diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index 6dac5e4..06e6b0c 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -1,7 +1,7 @@ DROP TABLE IF EXISTS `hk_change_log`; CREATE TABLE `hk_change_log` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `biz_key` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '业务key', + `biz_id` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '业务key', `biz_type` int(11) NOT NULL COMMENT '业务类型：1规则变更；2worker变更', `from_str` varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '原始值', `to_str` varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '目标值', @@ -31,24 +31,6 @@ CREATE TABLE `hk_user` ( ) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; - -DROP TABLE IF EXISTS `hk_key_rule`; -CREATE TABLE `hk_key_rule` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `key_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'key', - `prefix` int(11) NOT NULL COMMENT '是否前缀：1是；0否', - `intervals` int(11) NOT NULL COMMENT '间隔时间（秒）', - `threshold` int(11) NOT NULL COMMENT '阈值', - `duration` int(11) NOT NULL DEFAULT 60 COMMENT '缓存时间', - `app_name` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '所属appName', - `state` int(11) NOT NULL COMMENT '状态：1可用；-1删除', - `update_user` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT '修改人', - `update_time` datetime(0) NOT NULL ON UPDATE CURRENT_TIMESTAMP(0) COMMENT '修改时间', - `version` int(11) NOT NULL DEFAULT 0 COMMENT '数据版本', - PRIMARY KEY (`id`) USING BTREE -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; - - DROP TABLE IF EXISTS `hk_key_record`; CREATE TABLE `hk_key_record` ( `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', @@ -66,23 +48,6 @@ CREATE TABLE `hk_key_record` ( ) ENGINE = InnoDB AUTO_INCREMENT = 3 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; -DROP TABLE IF EXISTS `hk_change_log`; -CREATE TABLE `hk_change_log` ( - `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', - `biz_key` varchar(128) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '业务ID', - `biz_type` int(11) NOT NULL COMMENT '业务类型：1规则变更；2worker变更', - `from_str` varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '原始值', - `to_str` varchar(1024) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '目标值', - `app_name` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '数据所属APP', - `update_user` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '修改人', - `create_time` datetime(0) NOT NULL ON UPDATE CURRENT_TIMESTAMP(0) COMMENT '创建时间', - `uuid` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '防重ID', - PRIMARY KEY (`id`) USING BTREE, - UNIQUE INDEX `uniq_key`(`uuid`) USING BTREE COMMENT '唯一索引' -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; - - - DROP TABLE IF EXISTS `hk_key_timely`; CREATE TABLE `hk_key_timely` ( `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', @@ -97,20 +62,6 @@ CREATE TABLE `hk_key_timely` ( ) ENGINE = InnoDB AUTO_INCREMENT = 8 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; -DROP TABLE IF EXISTS `hk_app_info`; -CREATE TABLE `hk_app_info` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', - `app_name` varchar(60) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '应用名称', - `principal_name` varchar(20) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '负责人', - `principal_phone` varchar(11) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '负责人手机号', - `app_desc` varchar(64) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT '应用描述', - `create_time` datetime NOT NULL ON UPDATE CURRENT_TIMESTAMP(0) COMMENT '创建/接入时间', - PRIMARY KEY (`id`) USING BTREE, - UNIQUE INDEX `uniq_key`(`app_name`) USING BTREE COMMENT '唯一索引' -) ENGINE = InnoDB AUTO_INCREMENT = 8 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; - - - CREATE TABLE `hk_statistics` ( `id` int(11) NOT NULL AUTO_INCREMENT, `key_name` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'keyName', @@ -128,21 +79,6 @@ CREATE TABLE `hk_statistics` ( ) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = Compact; -CREATE TABLE `hk_receive_count` ( - `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键ID', - `worker_name` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT 'key名称', - `receive_count` int(11) NOT NULL COMMENT '接收数', - `hours` int(20) NOT NULL COMMENT '小时数', - `minutes` bigint(20) NOT NULL COMMENT '分钟数', - `seconds` bigint(20) NOT NULL COMMENT '秒数', - `uuid` varchar(150) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '防重ID', - `create_time` datetime(0) NOT NULL COMMENT '修改时间', - PRIMARY KEY (`id`) USING BTREE, - UNIQUE INDEX `uniq_uuid`(`uuid`) USING BTREE COMMENT '防重ID' -) ENGINE = InnoDB AUTO_INCREMENT = 1 CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = Compact; - - - CREATE TABLE `hk_rules` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '主键', `rules` varchar(5000) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL COMMENT '规则JSON', -- Gitee From 4c315c9f746817da59997e559827c268465fa589 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Mon, 20 Jul 2020 11:02:04 +0800 Subject: [PATCH 41/51] update README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1a7808a..62ab375 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ https://mp.weixin.qq.com/s/xOzEj5HtCeh_ezHDPHw6Jw ### 界面效果 ![输入图片说明](https://images.gitee.com/uploads/images/2020/0622/163805_0aa68d4b_303698.png "屏幕截图.png") ### 微信群 -![输入图片说明](https://images.gitee.com/uploads/images/2020/0713/131433_c172945c_303698.jpeg "11.jpeg") +![输入图片说明](https://images.gitee.com/uploads/images/2020/0720/110150_fee90b3e_303698.jpeg "11.jpeg") #### 安装教程 -- Gitee From 117646bdfbc0b5988b07a9c3b54430184e044262 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Thu, 23 Jul 2020 16:51:03 +0800 Subject: [PATCH 42/51] update README.md. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 62ab375..4e25ee7 100644 --- a/README.md +++ b/README.md @@ -62,8 +62,8 @@ https://mp.weixin.qq.com/s/xOzEj5HtCeh_ezHDPHw6Jw ### 界面效果 ![输入图片说明](https://images.gitee.com/uploads/images/2020/0622/163805_0aa68d4b_303698.png "屏幕截图.png") -### 微信群 -![输入图片说明](https://images.gitee.com/uploads/images/2020/0720/110150_fee90b3e_303698.jpeg "11.jpeg") +### 加我微信拉入群 +![输入图片说明](https://images.gitee.com/uploads/images/2020/0723/165039_67a40b9f_303698.jpeg "WechatIMG77.jpeg") #### 安装教程 -- Gitee From 97c399083303be2bd263372ad22ae25e18952a61 Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Mon, 27 Jul 2020 14:34:23 +0800 Subject: [PATCH 43/51] update README.md. --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 4e25ee7..53a9a2d 100644 --- a/README.md +++ b/README.md @@ -192,5 +192,11 @@ Object JdHotKeyStore.getValue(String key) } +### 测试环境 + +我司为方便大家快速接入试用、查看hotkey效果，提供了1台公网etcd机器（16c16g）、2台worker机器（8c12g），供快速接入测试。 + + +etcd地址为： **http://open-etcd.jd.com:2000** ，可以在下载项目后，在sample项目、dashboard项目里yml文件修改etcd连接地址为该地址，然后启动dashboard，进入界面控制台，即可看到当前连接的worker，之后可以在sample项目里，进行hotkey测试。 -- Gitee From 332d19cd6a889ddca375115bf6b9b6408c36dcfe Mon Sep 17 00:00:00 2001 From: tianyaleixiaowu <272551766@qq.com> Date: Mon, 27 Jul 2020 14:35:38 +0800 Subject: [PATCH 44/51] update README.md. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 53a9a2d..68ebdf7 100644 --- a/README.md +++ b/README.md @@ -199,4 +199,4 @@ Object JdHotKeyStore.getValue(String key) etcd地址为： **http://open-etcd.jd.com:2000** ，可以在下载项目后，在sample项目、dashboard项目里yml文件修改etcd连接地址为该地址，然后启动dashboard，进入界面控制台，即可看到当前连接的worker，之后可以在sample项目里，进行hotkey测试。 - +测试前，注意在控制台新建自己的app规则。 -- Gitee From 6c9d63d83f0a7d56535ba8eb1c9b37b487b597a5 Mon Sep 17 00:00:00 2001 From: wuweifeng10 Date: Tue, 28 Jul 2020 13:30:17 +0800 Subject: [PATCH 45/51] =?UTF-8?q?yml=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dashboard/src/main/resources/application.yml | 23 +++++++------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/dashboard/src/main/resources/application.yml b/dashboard/src/main/resources/application.yml index bc4609f..2e1b78e 100644 --- a/dashboard/src/main/resources/application.yml +++ b/dashboard/src/main/resources/application.yml @@ -1,5 +1,5 @@ server : - port : 8088 + port : 8081 servlet : context-path : / spring : @@ -27,26 +27,19 @@ spring : time-zone: GMT+8 date-format: yyyy-MM-dd HH:mm:ss datasource: - username: root - password: JRTEST - url: jdbc:mysql://172.24.7.182:3306/test1?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC + username: ${MYSQL_USER:root} + password: ${MYSQL_PASS:JRTEST} + url: jdbc:mysql://${MYSQL_HOST:localhost}:3306/luck?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC&useTimezone=true&serverTimezone=GMT%2B8 driver-class-name: com.mysql.cj.jdbc.Driver pagehelper: helperDialect: mysql reasonable: true supportMethodsArguments: true params: count=countSql -mybatis: - configuration: - log-impl: org.apache.ibatis.logging.stdout.StdOutImpl +#mybatis: +# configuration: +# log-impl: org.apache.ibatis.logging.stdout.StdOutImpl etcd: server: ${etcdServer:http://127.0.0.1:2379} -erp: - enabled: false - ssoAppKey: fangzhou - ssoAppToken: 3f48d7ade61940f2ba22079ebc273fb8 - loginUrl: https://ssa.jd.com/sso/login - excludePath: /login,/assets,/bootstarp,/common,/htmlformat,/images,/jn,/jquery,/user/login,login - ssoAppUrl: http://ssa.jd.com - defaultPwd: 123456 + -- Gitee From b7c2649db52389cfc2cf2d9707d1f842282e25df Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 28 Jul 2020 14:02:27 +0800 Subject: [PATCH 46/51] master fix --- .../hotkey/dashboard/interceptor/JwtInterceptor.java | 5 ++--- .../hotkey/dashboard/service/impl/UserServiceImpl.java | 1 - dashboard/src/main/resources/db.sql | 5 +++++ .../static/admin/assets/js/bootstrap/js/base_list.js | 2 +- .../main/resources/templates/admin/common/html/topMenu.html | 4 ++-- dashboard/src/main/resources/templates/admin/main.html | 2 +- 6 files changed, 11 insertions(+), 8 deletions(-) diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java index 173a465..02bc53e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/interceptor/JwtInterceptor.java @@ -29,9 +29,8 @@ public class JwtInterceptor extends HandlerInterceptorAdapter{ } String header = request.getHeader("x-requested-with"); if(!StringUtils.isEmpty(header) && "XMLHttpRequest".endsWith(header) && request.getMethod().equals(Constant.POST)){ - final String authHeader = JwtTokenUtil.getAuthHeader(request); - if (StringUtils.isEmpty(authHeader) - || !authHeader.startsWith(JwtTokenUtil.TOKEN_PREFIX)) { + String authHeader = request.getHeader(JwtTokenUtil.AUTH_HEADER_KEY); + if (StringUtils.isEmpty(authHeader) || !authHeader.startsWith(JwtTokenUtil.TOKEN_PREFIX)) { throw new BizException(ResultEnum.NO_LOGIN); } final String token = authHeader.substring(2); diff --git a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java index 3d39e79..d81f51e 100644 --- a/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java +++ b/dashboard/src/main/java/com/jd/platform/hotkey/dashboard/service/impl/UserServiceImpl.java @@ -108,5 +108,4 @@ public class UserServiceImpl implements UserService { return new PageInfo<>(users); } - } diff --git a/dashboard/src/main/resources/db.sql b/dashboard/src/main/resources/db.sql index 06e6b0c..98fbd82 100644 --- a/dashboard/src/main/resources/db.sql +++ b/dashboard/src/main/resources/db.sql @@ -31,6 +31,11 @@ CREATE TABLE `hk_user` ( ) ENGINE = InnoDB AUTO_INCREMENT = 2 CHARACTER SET = utf8 COLLATE = utf8_bin ROW_FORMAT = Compact; +-- pwd: 123456 +INSERT INTO `hk_user` VALUES (2, 'admin', 'admin', 'e10adc3949ba59abbe56e057f20f883e', '1888888', 'ADMIN', '', '2020-07-28 14:01:03', 1); + + + DROP TABLE IF EXISTS `hk_key_record`; CREATE TABLE `hk_key_record` ( `id` bigint(20) NOT NULL AUTO_INCREMENT COMMENT '主键', diff --git a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js index 05eb56e..59b7aac 100644 --- a/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js +++ b/dashboard/src/main/resources/static/admin/assets/js/bootstrap/js/base_list.js @@ -53,7 +53,7 @@ }, onLoadError: function(status){ //加载失败时执行 let token = getCookie("token"); - if(status === 500 && ( token == "undefined" || token =="")){ + if(status === 1000 && ( token == "undefined" || token =="")){ top.location.href = '/user/login'; } console.info("加载数据失败"); diff --git a/dashboard/src/main/resources/templates/admin/common/html/topMenu.html b/dashboard/src/main/resources/templates/admin/common/html/topMenu.html index bc89744..b5f55ac 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/topMenu.html +++ b/dashboard/src/main/resources/templates/admin/common/html/topMenu.html @@ -135,11 +135,11 @@   Help --> -

  • + diff --git a/dashboard/src/main/resources/templates/admin/main.html b/dashboard/src/main/resources/templates/admin/main.html index ee7b7fe..75e714a 100644 --- a/dashboard/src/main/resources/templates/admin/main.html +++ b/dashboard/src/main/resources/templates/admin/main.html @@ -179,7 +179,7 @@ }, error: function(result){ //加载失败时执行 let token = getCookie("token"); - if(result.status == 500 && ( token == "undefined" || token =="")){ + if(result.status == 1000 && ( token == "undefined" || token =="")){ top.location.href = '/user/login'; } console.info("加载数据失败"); -- Gitee From 019c957e00372eb976cbdf6c19d93043674da113 Mon Sep 17 00:00:00 2001 From: liyunfeng31 Date: Tue, 28 Jul 2020 14:14:06 +0800 Subject: [PATCH 47/51] master fix --- .../templates/admin/common/html/footer.html | 4 +-- .../templates/admin/common/html/title.html | 4 +-- .../templates/admin/common/html/topMenu.html | 25 +++++++++---------- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/dashboard/src/main/resources/templates/admin/common/html/footer.html b/dashboard/src/main/resources/templates/admin/common/html/footer.html index 662d4f3..703a359 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/footer.html +++ b/dashboard/src/main/resources/templates/admin/common/html/footer.html @@ -10,8 +10,8 @@ - 2020 京东零售-平台业务中心-平台业务研发部-基础业务研发部 版权所有 | 使用支持：武伟峰 | 咚咚：85134158 -
    接入文档：https://cf.jd.com/pages/viewpage.action?pageId=277756998 +
    diff --git a/dashboard/src/main/resources/templates/admin/common/html/title.html b/dashboard/src/main/resources/templates/admin/common/html/title.html index 08cf7c8..aa2fcaf 100644 --- a/dashboard/src/main/resources/templates/admin/common/html/title.html +++ b/dashboard/src/main/resources/templates/admin/common/html/title.html @@ -36,8 +36,8 @@
    Widget
    @@ -34,33 +33,33 @@