diff --git a/ci/centos7_python3_prepare.sh b/ci/centos7_python3_prepare.sh index 58dd87f64e9f882ef5a763d7f46bafd9b47684e2..348e0fb5739e93a8f69dcf49b172ba567ed3d5b4 100644 --- a/ci/centos7_python3_prepare.sh +++ b/ci/centos7_python3_prepare.sh @@ -50,6 +50,7 @@ pip3 install Pyinstaller pip3 install urllib3 pip3 install requests pip3 install paramiko +pip3 install psutil pip3 install wget pip3 install pyyaml diff --git a/common/devkit_utils/devkit_client.py b/common/devkit_utils/devkit_client.py index 7e806c9ff41a584f8c193687c0ed9d3bbc198b37..3d972bc7c2a598e7ba247900785f22623615b161 100644 --- a/common/devkit_utils/devkit_client.py +++ b/common/devkit_utils/devkit_client.py @@ -65,7 +65,20 @@ class DevKitClient: logging.exception(ex) pass - def upload_report(self, file_path): + def upload_report_by_force(self, file_path): + ret = self.__upload_report(file_path) + if ret.status_code == requests.codes.ok: + return + if ret.json().get("code", "") == "JavaPerf.Upload.Recording.RecordingReachLimit": + records = self.__get_record_list() + task_id, create_time = "", "999999999999999999999999999999" + for o in records.json().get("members", []): + if float(o["createTime"]) < float(create_time): + task_id, create_time = o["id"], o["createTime"] + self.__delete_report(task_id) + self.__upload_report(file_path) + + def __upload_report(self, file_path): try: data = dict({"file": (os.path.basename(file_path), open(file_path, "rb").read())}) except OSError as e: @@ -80,7 +93,7 @@ class DevKitClient: else: return requests.post(url=url, headers=_header, data=encoded_data[0], verify=False, proxies=self.NO_PROXY) - def get_record_list(self): + def __get_record_list(self): url = f"https://{self.ip}:{self.port}/plugin/api/v1.0/java_perf/api/records/user/" data = {"userId": self.user_id} if self.use_proxy: @@ -88,31 +101,9 @@ class DevKitClient: else: return requests.post(url=url, json=data, headers=self.header, verify=False, proxies=self.NO_PROXY) - def delete_report(self, task_id): + def __delete_report(self, task_id): url = f"https://{self.ip}:{self.port}/plugin/api/v1.0/java_perf/api/records/{task_id}/" if self.use_proxy: requests.delete(url=url, headers=self.header, verify=False) else: requests.delete(url=url, headers=self.header, verify=False, proxies=self.NO_PROXY) - - def upload_report_by_force(self, file_path): - ret = self.upload_report(file_path) - if ret.status_code == requests.codes.ok: - return - if ret.json().get("code", "") == "JavaPerf.Upload.Recording.RecordingReachLimit": - records = self.get_record_list() - task_id, create_time = "", "999999999999999999999999999999" - for o in records.json().get("members", []): - if float(o["createTime"]) < float(create_time): - task_id, create_time = o["id"], o["createTime"] - self.delete_report(task_id) - self.upload_report(file_path) - - -if __name__ == "__main__": - try: - d = DevKitClient("172.39.173.2", "8086", "devadmin", "Huawei12#$") - d.upload_report_by_force("/home/panlonglong/Downloads/Main(136462)") - d.logout() - except Exception as e: - print(str(e)) diff --git a/common/devkit_utils/pyinstaller_utils.py b/common/devkit_utils/pyinstaller_utils.py index 91f578bcd2b2e90b671e9d36e432a031826aceb6..bde176c7243f3738f449bfdb54cf92809b9e81e0 100644 --- a/common/devkit_utils/pyinstaller_utils.py +++ b/common/devkit_utils/pyinstaller_utils.py @@ -1,22 +1,45 @@ +import copy import os.path import sys -def check_is_running_in_pyinstaller_bundle(): - """ - 判断是否在pyinstaller - """ - if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): - return True - else: - return False +class PyInstallerUtils: + ORI_ENV = None + LD_LIBRARY_PATH_ORIG = "LD_LIBRARY_PATH_ORIG" + LD_LIBRARY_PATH = "LD_LIBRARY_PATH" + @classmethod + def check_is_running_in_pyinstaller_bundle(cls): + """ + 判断是否在pyinstaller + """ + if getattr(sys, "frozen", False) and hasattr(sys, "_MEIPASS"): + return True + else: + return False -def obtain_root_path(root_path): - """ - 获取rootpath,当在pyinstaller中时，为父目录，否则为传入的参数 - """ - if check_is_running_in_pyinstaller_bundle(): - return os.path.dirname(os.path.dirname(sys.executable)) - else: - return root_path + @classmethod + def obtain_root_path(cls, root_path): + """ + 获取rootpath,当在pyinstaller中时，为父目录，否则为传入的参数 + """ + if cls.check_is_running_in_pyinstaller_bundle(): + return os.path.dirname(os.path.dirname(sys.executable)) + else: + return root_path + + @classmethod + def get_env(cls): + """ + 返回的env,不允许修改 + """ + if cls.ORI_ENV: + return cls.ORI_ENV + env_copy = copy.deepcopy(os.environ) + if cls.check_is_running_in_pyinstaller_bundle(): + if cls.LD_LIBRARY_PATH_ORIG in env_copy: + env_copy[cls.LD_LIBRARY_PATH] = env_copy.get(cls.LD_LIBRARY_PATH_ORIG) + else: + env_copy.pop(cls.LD_LIBRARY_PATH) + cls.ORI_ENV = env_copy + return cls.ORI_ENV diff --git a/common/devkit_utils/shell_tools.py b/common/devkit_utils/shell_tools.py index 07324f7074c360a6ab52de074ac306e6c65564f6..60fb33bf0aaa1b528d4a2dbf481ce5b004d262fc 100644 --- a/common/devkit_utils/shell_tools.py +++ b/common/devkit_utils/shell_tools.py @@ -3,6 +3,8 @@ import shlex import subprocess import typing +from devkit_utils.pyinstaller_utils import PyInstallerUtils + class ExecutionOutcome: def __init__(self, return_code=0, out=None, err=None): @@ -14,17 +16,17 @@ class ExecutionOutcome: return "code:%s\n out:%s\n err:%s\n" % (self.return_code, self.out, self.err) -def exec_shell(command: str, is_shell=False, timeout=30) -> ExecutionOutcome: +def exec_shell(command: str, is_shell=False, timeout=30, env=PyInstallerUtils.get_env()) -> ExecutionOutcome: """ 执行命令，返回执行结果 """ try: if is_shell: - child = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stdin=None, + child = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stdin=None, env=env, stderr=subprocess.PIPE, encoding="utf-8", universal_newlines=True, shell=is_shell) else: - child = subprocess.Popen(shlex.split(command), close_fds=True, stdout=subprocess.PIPE, stdin=None, + child = subprocess.Popen(shlex.split(command), close_fds=True, stdout=subprocess.PIPE, stdin=None, env=env, stderr=subprocess.PIPE, encoding="utf-8", universal_newlines=True) except Exception as ex: diff --git a/component/DevKitTester/JFRParser/pom.xml b/component/DevKitTester/JFRParser/pom.xml index 36289eebb1d5a7920afed2e5475d41e31a8b8596..a2ac452ffcf5d092cf3f372e288592cdf203e93e 100644 --- a/component/DevKitTester/JFRParser/pom.xml +++ b/component/DevKitTester/JFRParser/pom.xml @@ -30,6 +30,11 @@ jackson-databind 2.16.1 + + org.apache.commons + commons-math3 + 3.6.1 + args4j args4j diff --git a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/CpuInfo.java b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/CpuInfo.java index b37e5d436033b337b0db8902900b02d1e70c7d7b..3917bc146df200fb0d3e350ad2d9ae881e3ec44b 100644 --- a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/CpuInfo.java +++ b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/CpuInfo.java @@ -7,13 +7,13 @@ public class CpuInfo { @JsonProperty("t") private long startTime; @JsonProperty("s") - private float jvmSystem; + private Float jvmSystem; @JsonProperty("u") - private float jvmUser; + private Float jvmUser; @JsonProperty("m") - private float machineTotal; + private Float machineTotal; - public CpuInfo(long startTime, float jvmSystem, float jvmUser, float machineTotal) { + public CpuInfo(long startTime, Float jvmSystem, Float jvmUser, Float machineTotal) { this.startTime = startTime; this.jvmSystem = jvmSystem; this.jvmUser = jvmUser; diff --git a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/JmeterReportSummary.java b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/JmeterReportSummary.java index 01417eb367cd4f302db9643a32e9adb199fa337d..e677e1d0a137a230ac8720185459e062da06ec25 100644 --- a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/JmeterReportSummary.java +++ b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/JmeterReportSummary.java @@ -1,5 +1,8 @@ package com.huawei.devkit.pipeline.bo; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.huawei.devkit.pipeline.strategy.DoubleSerialize; + public class JmeterReportSummary { private String label; @@ -8,21 +11,29 @@ public class JmeterReportSummary { private long failSamples; + @JsonSerialize(using = DoubleSerialize.class) private double averageLatency; - private long minLatency; + @JsonSerialize(using = DoubleSerialize.class) + private double minLatency; - private long maxLatency; + @JsonSerialize(using = DoubleSerialize.class) + private double maxLatency; - private long median; + @JsonSerialize(using = DoubleSerialize.class) + private double median; - private long latency99; + @JsonSerialize(using = DoubleSerialize.class) + private double latency99; - private long latency95; + @JsonSerialize(using = DoubleSerialize.class) + private double latency95; - private long latency90; + @JsonSerialize(using = DoubleSerialize.class) + private double latency90; - private long throughput; + @JsonSerialize(using = DoubleSerialize.class) + private double throughput; public JmeterReportSummary(String label) { this.label = label; @@ -68,59 +79,59 @@ public class JmeterReportSummary { this.averageLatency = averageLatency; } - public long getMinLatency() { + public double getMinLatency() { return minLatency; } - public void setMinLatency(long minLatency) { + public void setMinLatency(double minLatency) { this.minLatency = minLatency; } - public long getMaxLatency() { + public double getMaxLatency() { return maxLatency; } - public void setMaxLatency(long maxLatency) { + public void setMaxLatency(double maxLatency) { this.maxLatency = maxLatency; } - public long getMedian() { + public double getMedian() { return median; } - public void setMedian(long median) { + public void setMedian(double median) { this.median = median; } - public long getLatency99() { + public double getLatency99() { return latency99; } - public void setLatency99(long latency99) { + public void setLatency99(double latency99) { this.latency99 = latency99; } - public long getLatency95() { + public double getLatency95() { return latency95; } - public void setLatency95(long latency95) { + public void setLatency95(double latency95) { this.latency95 = latency95; } - public long getLatency90() { + public double getLatency90() { return latency90; } - public void setLatency90(long latency90) { + public void setLatency90(double latency90) { this.latency90 = latency90; } - public long getThroughput() { + public double getThroughput() { return throughput; } - public void setThroughput(long throughput) { + public void setThroughput(double throughput) { this.throughput = throughput; } } diff --git a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/JmeterTPS.java b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/JmeterTPS.java index 2b5d1c2d64d5247bffe3fe17bf6adf44bd933aff..d406ffc79fcd64e01c92a03aa9f268693be657b4 100644 --- a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/JmeterTPS.java +++ b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/JmeterTPS.java @@ -14,4 +14,19 @@ public class JmeterTPS { this.tps = tps; } + public Long getStartTime() { + return startTime; + } + + public void setStartTime(Long startTime) { + this.startTime = startTime; + } + + public int getTps() { + return tps; + } + + public void setTps(int tps) { + this.tps = tps; + } } diff --git a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/MemInfo.java b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/MemInfo.java index eef71220312b69535628d7225a6e686e45029ba0..2287cb85bc40e80a77bc8fe9e75d6929880b0b27 100644 --- a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/MemInfo.java +++ b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/MemInfo.java @@ -6,13 +6,13 @@ public class MemInfo { @JsonProperty("t") private long startTime; @JsonProperty("c") - private long committedSize; + private Long committedSize; @JsonProperty("r") - private long reservedSize; + private Long reservedSize; @JsonProperty("u") - private long heapUsed; + private Long heapUsed; - public MemInfo(long startTime, long committedSize, long reservedSize, long heapUsed) { + public MemInfo(long startTime, Long committedSize, Long reservedSize, Long heapUsed) { this.startTime = startTime; this.committedSize = committedSize; this.reservedSize = reservedSize; diff --git a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/PerformanceTestResult.java b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/PerformanceTestResult.java index 226cb208ed465bf56f61646dc1abee656e1d3c4f..445a93f82bf050927011ee7b09bb5b3fcb3eaffe 100644 --- a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/PerformanceTestResult.java +++ b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/bo/PerformanceTestResult.java @@ -9,6 +9,8 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; public class PerformanceTestResult { @@ -73,6 +75,22 @@ public class PerformanceTestResult { this.toSimpleObject(this.frtMap, this.frt, JmeterRT.class); this.toSimpleObject(this.tpsMap, this.tps, JmeterTPS.class); } + List allStartTime = memoryMap.values().stream() + .flatMap(innerMap -> innerMap.values().stream()) + .flatMap(List::stream) + .map(MemInfo::getStartTime) + .distinct() + .sorted() + .collect(Collectors.toList()); + this.insertNullInMemoryMap(allStartTime); + List allCpuStartTime = cpuMap.values().stream() + .flatMap(innerMap -> innerMap.values().stream()) + .flatMap(List::stream) + .map(CpuInfo::getStartTime) + .distinct() + .sorted() + .collect(Collectors.toList()); + this.insertNullInCpuMap(allCpuStartTime); this.toSimpleObject2(this.memoryMap, this.memory, MemInfo.class); this.toSimpleObject2(this.cpuMap, this.cpu, CpuInfo.class); } @@ -193,6 +211,52 @@ public class PerformanceTestResult { } } + private void insertNullInMemoryMap(List allStartTime) { + for (String key : this.memoryMap.keySet()) { + Map> oldMap = this.memoryMap.get(key); + Map> newMap = new HashMap<>(); + for (Map.Entry> entry : oldMap.entrySet()) { + List memInfos = this.insertNullInMemoryMap(entry.getValue(), allStartTime); + newMap.put(entry.getKey(), memInfos); + } + this.memoryMap.put(key, newMap); + } + } + + private List insertNullInMemoryMap(List target, List allStartTime) { + List finalMem = new ArrayList<>(allStartTime.size()); + Map collected = target.stream() + .collect(Collectors.toMap(MemInfo::getStartTime, Function.identity())); + for (Long key : allStartTime) { + MemInfo info = collected.get(key); + finalMem.add(Objects.requireNonNullElseGet(info, () -> new MemInfo(key, null, null, null))); + } + return finalMem; + } + + private void insertNullInCpuMap(List allStartTime) { + for (String key : this.cpuMap.keySet()) { + Map> oldMap = this.cpuMap.get(key); + Map> newMap = new HashMap<>(); + for (Map.Entry> entry : oldMap.entrySet()) { + List memInfos = this.insertNullInCpuMap(entry.getValue(), allStartTime); + newMap.put(entry.getKey(), memInfos); + } + this.cpuMap.put(key, newMap); + } + } + + private List insertNullInCpuMap(List target, List allStartTime) { + List finalMem = new ArrayList<>(allStartTime.size()); + Map collected = target.stream() + .collect(Collectors.toMap(CpuInfo::getStartTime, Function.identity())); + for (Long key : allStartTime) { + CpuInfo info = collected.get(key); + finalMem.add(Objects.requireNonNullElseGet(info, () -> new CpuInfo(key, null, null, null))); + } + return finalMem; + } + private void toSimpleObject2(Map>> origin, Map>>> target, Class clazz) { for (Map.Entry>> entry : origin.entrySet()) { diff --git a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/parser/JFRParser.java b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/parser/JFRParser.java index ca3bcb33bf0afc76a54476269fc80ca03e18297a..46d3347496a65a5935048b216e868026e2447cca 100644 --- a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/parser/JFRParser.java +++ b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/parser/JFRParser.java @@ -15,8 +15,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; public class JFRParser { public static Long ALL = -1L; @@ -66,7 +70,16 @@ public class JFRParser { Map> cpuMap = result.getCpuMap().get(nodeIP); cpuMap.put(fileName, cpuInfos); Map> memoryMap = result.getMemoryMap().get(nodeIP); - memoryMap.put(fileName, memInfos); + Set allStartTime = new HashSet<>(); + memoryMap.put(fileName, memInfos.stream().filter(item -> { + if (allStartTime.contains(item.getStartTime())) { + return false; + } else { + allStartTime.add(item.getStartTime()); + return true; + } + }) + .sorted(Comparator.comparingLong(MemInfo::getStartTime)).collect(Collectors.toList())); for (LatencyTopInfo latencyTop : top10) { result.getFlame().put(latencyTop.getKey(), latencyTop.getFlame()); } diff --git a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/parser/JmeterResultParser.java b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/parser/JmeterResultParser.java index f277eaff1dc5ba05440c8f106116e6ef191dfdd1..9ab9bd597c8b900d1f279c18328b2ba74c81253a 100644 --- a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/parser/JmeterResultParser.java +++ b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/parser/JmeterResultParser.java @@ -7,6 +7,7 @@ import com.huawei.devkit.pipeline.bo.JmeterTPS; import com.huawei.devkit.pipeline.bo.PerformanceTestResult; import com.huawei.devkit.pipeline.constants.JFRConstants; import com.huawei.devkit.pipeline.utils.JmeterResultTransfer; +import org.apache.commons.math3.stat.descriptive.DescriptiveStatistics; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -84,12 +85,12 @@ public class JmeterResultParser { long start = startTime; long end = startTime + JFRConstants.MS_1000; - + DescriptiveStatistics statistics = new DescriptiveStatistics(); boolean exist = false; for (int i = 0; start < endTime; start += JFRConstants.MS_1000, end += JFRConstants.MS_1000) { - while (i < results.size() && results.get(i).getStartTime() >= start && results.get(i).getStartTime() < end) { summary.samplesIncrease(); + statistics.addValue(results.get(i).getLatency()); latencyTotalPerSec += results.get(i).getLatency(); latencyTotal += results.get(i).getLatency(); samplePer++; @@ -115,10 +116,16 @@ public class JmeterResultParser { latencyFailPerSec = 0; exist = false; } - summary.setThroughput(summary.getSamples() * JFRConstants.MS_TO_S / - getThroughputTime(startTime, results.get(results.size() - 1).getStartTime())); + summary.setMinLatency(statistics.getMin()); + summary.setMaxLatency(statistics.getMax()); + summary.setMedian(statistics.getPercentile(50)); + summary.setLatency90(statistics.getPercentile(90)); + summary.setLatency95(statistics.getPercentile(95)); + summary.setLatency99(statistics.getPercentile(99)); + JmeterResult lastResult = results.get(results.size() - 1); + summary.setThroughput(summary.getSamples() * (double) JFRConstants.MS_TO_S / + getThroughputTime(startTime, lastResult.getStartTime() + lastResult.getLatency())); summary.setAverageLatency(latencyTotal / (double) summary.getSamples()); - this.filledSummary(results); } public JmeterReportSummary getSummary() { diff --git a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/strategy/DoubleSerialize.java b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/strategy/DoubleSerialize.java new file mode 100644 index 0000000000000000000000000000000000000000..6ae65739d2350c106f5fc5db4bf3c091d3c9dd21 --- /dev/null +++ b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/strategy/DoubleSerialize.java @@ -0,0 +1,22 @@ +package com.huawei.devkit.pipeline.strategy; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; +import java.text.DecimalFormat; + +public class DoubleSerialize extends JsonSerializer { + + public static final DecimalFormat DF = new DecimalFormat("#0.00"); + + @Override + public void serialize(Double value, JsonGenerator gen, SerializerProvider serializers) + throws IOException, JsonProcessingException { + if (value != null) { + gen.writeString(DF.format(value)); + } + } +} \ No newline at end of file diff --git a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/utils/SimplifyResponse.java b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/utils/SimplifyResponse.java index 188e2609bfc6894d610735e3ebb48e84876f2479..9841cd4cdceeca4bbb28c26b240822b34e80de9a 100644 --- a/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/utils/SimplifyResponse.java +++ b/component/DevKitTester/JFRParser/src/main/java/com/huawei/devkit/pipeline/utils/SimplifyResponse.java @@ -1,6 +1,7 @@ package com.huawei.devkit.pipeline.utils; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.huawei.devkit.pipeline.strategy.DoubleSerialize; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -26,9 +27,21 @@ public class SimplifyResponse { List list = new ArrayList<>(); result.put(field.getName(), list); try { - for (T item : before) { - list.add(field.get(item)); + if (!before.isEmpty() && field.get(before.get(0)) instanceof Double) { + for (T item : before) { + Object value = field.get(item); + if (Objects.isNull(value)) { + list.add(null); + } else { + list.add(DoubleSerialize.DF.format(value)); + } + } + } else { + for (T item : before) { + list.add(field.get(item)); + } } + } catch (IllegalAccessException e) { logger.error(e); } diff --git a/component/DevKitTester/JFRParser/src/test/java/com/huawei/devkit/pipeline/parser/JFRParserTest.java b/component/DevKitTester/JFRParser/src/test/java/com/huawei/devkit/pipeline/parser/JFRParserTest.java index 73da293556a9d82d9f9ad67dc2ee9696857767d0..5b819de4f8367fab7542da42232b915ca72f466e 100644 --- a/component/DevKitTester/JFRParser/src/test/java/com/huawei/devkit/pipeline/parser/JFRParserTest.java +++ b/component/DevKitTester/JFRParser/src/test/java/com/huawei/devkit/pipeline/parser/JFRParserTest.java @@ -91,6 +91,6 @@ public final class JFRParserTest { Assertions.assertEquals(result.getFlame().get(1713159117349L).getValue(), 18); Assertions.assertEquals(result.getFlame().get(-1L).getValue(), 1463); Assertions.assertEquals(result.getCpuMap().get(nodeIP).get("avrora.jfr").size(), 70); - Assertions.assertEquals(result.getMemoryMap().get(nodeIP).get("avrora.jfr").size(), 48); + Assertions.assertEquals(result.getMemoryMap().get(nodeIP).get("avrora.jfr").size(), 36); } } diff --git a/component/DevKitTester/JFRParser/src/test/java/com/huawei/devkit/pipeline/utils/SimplifyResponseTest.java b/component/DevKitTester/JFRParser/src/test/java/com/huawei/devkit/pipeline/utils/SimplifyResponseTest.java index cd33b026a4b5018fc0032b8e3685e9641ba6ce73..ab09715a3f20f7c12e29a8c9c5ea49bf01e6108f 100644 --- a/component/DevKitTester/JFRParser/src/test/java/com/huawei/devkit/pipeline/utils/SimplifyResponseTest.java +++ b/component/DevKitTester/JFRParser/src/test/java/com/huawei/devkit/pipeline/utils/SimplifyResponseTest.java @@ -34,6 +34,6 @@ public final class SimplifyResponseTest { jmeterRTList.add(new JmeterRT(4L, 4d)); Map> simplified = SimplifyResponse.simplify(jmeterRTList, JmeterRT.class); Assertions.assertEquals(simplified.get("responseTime").size(), 4); - Assertions.assertEquals(simplified.get("responseTime").toString(), List.of(1d, 2d, 3d, 4d).toString()); + Assertions.assertEquals(simplified.get("responseTime").toString(), List.of("1.00", "2.00", "3.00", "4.00").toString()); } } diff --git a/component/DevKitTester/devkit_tester/bin/entrance.py b/component/DevKitTester/devkit_tester/bin/entrance.py index 8c8a92e35c506c00dc1a9d599ccb7916898fdd00..c2f997f9b244052fe4384002b2fbc3c273c70946 100644 --- a/component/DevKitTester/devkit_tester/bin/entrance.py +++ b/component/DevKitTester/devkit_tester/bin/entrance.py @@ -1,4 +1,5 @@ import argparse +import copy import datetime import logging import os @@ -12,7 +13,7 @@ from devkit_utils import shell_tools from devkit_utils.devkit_client import DevKitClient from devkit_utils.error_coce import ErrorCodeEnum, ErrorCodeMsg from devkit_utils.log_config import config_log_ini -from devkit_utils.pyinstaller_utils import obtain_root_path +from devkit_utils.pyinstaller_utils import PyInstallerUtils from devkit_utils.transport_utils import SSHClientFactory from report.report import Report @@ -20,11 +21,12 @@ ROOT_PATH = os.path.dirname(os.path.dirname(__file__)) class JmeterCommand: - def __init__(self, origin_command): + def __init__(self, origin_command, java_home): self.origin_command: str = origin_command self.csv_file = None self.result_dir = None self.jmx_file = None + self.java_home = java_home def check_and_init_jmeter_cmd(self): if not self.origin_command: @@ -60,7 +62,9 @@ class JmeterCommand: else: break else: - return self.__check_param_resource() + self.__check_java_version() + self.__check_param_resource() + return raise Exception( "The command line is not supported。example: sh jmeter.sh -n -t /home/demo.jmt " "-l /home/jmeter/result.csv -e -o /home/jmeter/empty_dir/") @@ -76,6 +80,28 @@ class JmeterCommand: if not os.path.exists(self.jmx_file): raise Exception(f"the jmx file {self.jmx_file} is not exist") + def __check_java_version(self): + try: + if self.java_home: + if not os.path.exists(f"{self.java_home}/bin/java"): + raise Exception("The currently specified java home is incorrect base on -m parameter") + command = f"{self.java_home}/bin/java -version" + else: + command = f"java -version" + logging.info("command is %s", command) + outcome = shell_tools.exec_shell(command, is_shell=True, timeout=None) + output = outcome.err if "version" in outcome.err else outcome.out + version_line = output.split(" ")[2].strip('"') + version = version_line.split(".")[0] + except Exception as ex: + logging.exception(ex) + raise Exception("Please use the -m parameter to specify java home," + " and the java version is greater than or equal to 11") + else: + if int(version) < 11: + raise Exception("Please use the -m parameter to specify java home," + " and the java version is greater than or equal to 11") + class Distributor: SEVEN_DAYS = 60 * 60 * 24 * 7 @@ -100,19 +126,20 @@ class Distributor: file_utils.create_dir(self.data_path) self.template_path = os.path.join(self.root_path, "config") self.git_path = args.git_path - self.jmeter_command: JmeterCommand = JmeterCommand(args.jmeter_command) + self.java_home = args.java_home + self.output = args.output if args.output else self.data_path + self.jmeter_command: JmeterCommand = JmeterCommand(args.jmeter_command, self.java_home) self.jmeter_thread: typing.Optional[threading.Thread] = None self.enable_jmeter_command = True if args.jmeter_command else False + # 节点差异时间 self.node_time_gap = dict() + # 节点采集的JFR文件 self.node_jfr_path = dict() def distribute(self): # 清空本地jfr文件 file_utils.clear_dir(self.data_path) - # jmeter 命令校验 - self.jmeter_command.check_and_init_jmeter_cmd() - # 校验联通性 - self.__check_ips_connected() + self.__check() # 启动jmeter if self.enable_jmeter_command: self.__start_jmeter_thread() @@ -138,26 +165,49 @@ class Distributor: # 等待jmeter完成 if self.enable_jmeter_command: self.__generate_jmeter_data() - report = Report(report_path=self.data_path, template_path=self.template_path, + report = Report(report_dir=self.output, data_path=self.data_path, template_path=self.template_path, jmeter_report_path=self.jmeter_command.csv_file, git_path=self.git_path, devkit_tool_ip=self.devkit_ip, devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) else: - report = Report(report_path=self.data_path, template_path=self.template_path, + report = Report(report_dir=self.output, data_path=self.data_path, template_path=self.template_path, git_path=self.git_path, devkit_tool_ip=self.devkit_ip, devkit_tool_port=self.devkit_port, devkit_user_name=self.devkit_user) report.report() self.__print_result(jfr_names) + def __check(self): + # jmeter 命令校验 + self.jmeter_command.check_and_init_jmeter_cmd() + # 校验联通性 + self.__check_ips_connected() + # 校验output + if not os.path.exists(self.output) or not os.path.isdir(self.output): + raise Exception("the output path specified by the -o parameter is s not a directory or doesn't exist") + final_report = os.path.join(self.output, "devkit_performance_report.html") + if os.path.exists(final_report): + raise Exception( + "the output path specified by the -o parameter doesn't contain the file named " + "devkit_performance_report.html ") + if not os.access(self.output, os.R_OK | os.W_OK | os.X_OK): + raise Exception("The output path specified by the -o parameter does have no permissions") + def __generate_jmeter_data(self): time_gap = ','.join(f"{k}:{v}" for k, v in self.node_time_gap.items()) jfr_path = ','.join(f"{k}:{item}" for k, v in self.node_jfr_path.items() for item in v) - command = (f"bash {self.root_path}/bin/generate_jmeter_result.sh -o {self.data_path} " - f"-j {self.jmeter_command.csv_file} " - f"-n {time_gap} " - f"-f {jfr_path} ") + if self.java_home: + command = (f"export JAVA_HOME={self.java_home} && " + f"bash {self.root_path}/bin/generate_jmeter_result.sh -o {self.data_path} " + f"-j {self.jmeter_command.csv_file} " + f"-n {time_gap} " + f"-f {jfr_path} ") + else: + command = (f"bash {self.root_path}/bin/generate_jmeter_result.sh -o {self.data_path} " + f"-j {self.jmeter_command.csv_file} " + f"-n {time_gap} " + f"-f {jfr_path} ") logging.info("command is %s", command) - outcome = shell_tools.exec_shell(command, timeout=None) + outcome = shell_tools.exec_shell(command, is_shell=True, timeout=None) logging.info("return_code: %s", outcome.return_code) logging.info("error: %s", outcome.err) logging.info("out: %s", outcome.out) @@ -168,7 +218,12 @@ class Distributor: self.jmeter_thread.start() def __jmeter_start(self, command): - outcome = shell_tools.exec_shell(command, is_shell=True, timeout=self.SEVEN_DAYS) + if self.java_home: + new_env = copy.deepcopy(PyInstallerUtils.get_env()) + new_env["JAVA_HOME"] = self.java_home + outcome = shell_tools.exec_shell(command, is_shell=True, timeout=self.SEVEN_DAYS, env=new_env) + else: + outcome = shell_tools.exec_shell(command, is_shell=True, timeout=self.SEVEN_DAYS) logging.info("return_code: %s", outcome.return_code) logging.info("error: %s", outcome.err) logging.info("out: %s", outcome.out) @@ -360,7 +415,11 @@ def main(): help="git path") parser.add_argument("-j", "--jmeter-command", dest="jmeter_command", type=str, help="the command that start jmeter tests") - parser.set_defaults(root_path=obtain_root_path(ROOT_PATH)) + parser.add_argument("-m", "--java-home", dest="java_home", type=str, + help="the java home for parsing the jfr, the java version is greater than or equal to 11") + parser.add_argument("-o", "--output", dest="output", type=str, + help="the directory of the final report") + parser.set_defaults(root_path=PyInstallerUtils.obtain_root_path(ROOT_PATH)) parser.set_defaults(password="") args = parser.parse_args() config_log_ini(args.root_path, "devkit_tester") diff --git a/component/DevKitTester/devkit_tester/bin/report/report.py b/component/DevKitTester/devkit_tester/bin/report/report.py index 86df67d0957555bbf60fc91d03bc010be0ffffa0..bfad2c0136907be3ffc98d0f680a616e076c9f8a 100644 --- a/component/DevKitTester/devkit_tester/bin/report/report.py +++ b/component/DevKitTester/devkit_tester/bin/report/report.py @@ -1,4 +1,5 @@ import json +import logging import os import re import subprocess @@ -21,12 +22,13 @@ GIT_REPORT_DATA_LINE_NUM = 14 class Report: - def __init__(self, report_path="./", template_path="./", git_path="./", + def __init__(self, report_dir="./", data_path="./", template_path="./", git_path="./", jmeter_report_path=None, devkit_tool_ip="", devkit_tool_port="8086", devkit_user_name="devadmin"): - if not os.path.isdir(report_path): - raise Exception(f"Report path:{report_path} illegal.") - self.report_dir = report_path + if not os.path.isdir(report_dir): + raise Exception(f"Report path:{report_dir} illegal.") + self.report_dir = report_dir + self.data_path = data_path self.git_path = git_path self.template_path = template_path self.jmeter_report_path = jmeter_report_path @@ -42,7 +44,7 @@ class Report: html_lines[DEVKIT_REPORT_DATA_LINE_NUM] = "report_tb_data: {} \n".format(devkit_report_json) if self.jmeter_report_path: valid_page.append("'perf'") - jmeter_report = os.path.join(self.report_dir, "result.json") + jmeter_report = os.path.join(self.data_path, "result.json") with open(jmeter_report, 'r') as file: html_lines[JMETER_REPORT_DATA_LINE_NUM] = "perf: {},\n".format(file.read()) if self.git_path: @@ -51,6 +53,7 @@ class Report: html_lines[GIT_REPORT_DATA_LINE_NUM] = "git_tb_data: {},\n".format(git_log) html_lines[REPORT_VALID_LINE] = "const validPages = [{}];\n".format(",".join(valid_page)) final_report = os.path.join(self.report_dir, "devkit_performance_report.html") + logging.info("please view the report file : %s", os.path.realpath(final_report)) with open(final_report, "w") as file: file.writelines(html_lines) return final_report diff --git a/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py b/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py index b4f08b29e0b102191b4aef3a131ce7def49bd07e..4d8355d78bee3a9f279dffec40fe53a9097864ed 100644 --- a/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py +++ b/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py @@ -5,10 +5,12 @@ import os.path import shutil import time +import psutil + from devkit_utils import shell_tools, file_utils from devkit_utils.error_coce import ErrorCodeEnum from devkit_utils.log_config import config_log_ini -from devkit_utils.pyinstaller_utils import obtain_root_path +from devkit_utils.pyinstaller_utils import PyInstallerUtils ROOT_PATH = os.path.dirname(os.path.dirname(__file__)) @@ -43,6 +45,8 @@ class FlightRecordsFactory: self.response_file = os.path.join(root_path, "config/complete_the_upload.ini") self.now_date = datetime.datetime.now().strftime("%Y%m%d%H%M%S") file_utils.create_dir(self.dir_to_storage_jfr) + self.jcmd_path = None + self.user_is_root = False def start_sample(self): try: @@ -51,27 +55,52 @@ class FlightRecordsFactory: self.__init_pids() if len(self.pids) == 0: self.return_code = ErrorCodeEnum.NOT_FOUND_APPS + return elif not self.__check_jcmd(): self.return_code = ErrorCodeEnum.NOT_FOUND_JCMD + return + self.__init_user_is_root() logging.info("start_recorder") - self.__start_recorder() - if self.jfr_paths: - if self.wait_for_jmeter_stop: - self.__wait_for_jmeter_has_stopping() - self.__stop_recorder() - else: - time.sleep(self.duration) - before = datetime.datetime.now() - # 停止采集 - logging.info("check has stopped recorder") - while not self.__check_has_stopped_recorder() and (datetime.datetime.now() - before).seconds < 30: - time.sleep(1) + if self.user_is_root: + self.__start_sample_by_root() else: - logging.exception(f"The target application {self.apps} cannot be found or Operation not permitted") + self.__start_sample_by_common_user() finally: shell_tools.exec_shell(f"echo {self.return_code} >{self.response_file}", is_shell=True) logging.info("the current agent has been executed") + def __start_sample_by_root(self): + self.__start_recorder_by_root() + if self.jfr_paths: + if self.wait_for_jmeter_stop: + self.__wait_for_jmeter_has_stopping() + self.__stop_recorder_by_root() + else: + time.sleep(self.duration) + before = datetime.datetime.now() + # 停止采集 + logging.info("check has stopped recorder") + while not self.__check_has_stopped_recorder_by_root() and (datetime.datetime.now() - before).seconds < 30: + time.sleep(1) + else: + logging.exception(f"The target application {self.apps} cannot be found or Operation not permitted") + + def __start_sample_by_common_user(self): + self.__start_recorder() + if self.jfr_paths: + if self.wait_for_jmeter_stop: + self.__wait_for_jmeter_has_stopping() + self.__stop_recorder() + else: + time.sleep(self.duration) + before = datetime.datetime.now() + # 停止采集 + logging.info("check has stopped recorder") + while not self.__check_has_stopped_recorder() and (datetime.datetime.now() - before).seconds < 30: + time.sleep(1) + else: + logging.exception(f"The target application {self.apps} cannot be found or Operation not permitted") + def __wait_for_jmeter_has_stopping(self): before = datetime.datetime.now() while (datetime.datetime.now() - before).seconds < self.duration: @@ -99,16 +128,41 @@ class FlightRecordsFactory: for pid in pids: self.pids.append(TargetProcess(pid, app)) - @staticmethod - def __check_jcmd(): + def __check_jcmd(self): commander_to_check = "which jcmd" outcome = shell_tools.exec_shell(commander_to_check, is_shell=True) logging.info("check jcmd :%s", outcome) if outcome.return_code == 0: + self.jcmd_path = outcome.out.strip() return True else: return False + def __init_user_is_root(self): + if os.geteuid() == 0: + self.user_is_root = True + else: + self.user_is_root = False + + def __start_recorder_by_root(self): + logging.info(PyInstallerUtils.get_env()) + logging.info(os.environ) + for target in self.pids: + jfr_path = self.__jfr_name(target.name, target.pid) + username = psutil.Process(int(target.pid)).username() + command = (f"su - {username} -c '" + f"{self.jcmd_path} {target.pid} JFR.start settings={self.temporary_settings_path} " + f"duration={self.duration}s name={self.RECORD_NAME} filename={jfr_path}'") + logging.info(command) + outcome = shell_tools.exec_shell(command, is_shell=True) + logging.info(outcome) + if outcome.return_code == 0: + self.jfr_paths.append(jfr_path) + self.pids_to_start_recording.append(target) + # 移动到data目录下 + with open(file=os.path.join(self.root_path, "config/upload_sample.ini"), mode="w", encoding="utf-8") as file: + file.write(os.linesep.join(self.jfr_paths)) + def __start_recorder(self): for target in self.pids: jfr_path = self.__jfr_name(target.name, target.pid) @@ -124,12 +178,33 @@ class FlightRecordsFactory: with open(file=os.path.join(self.root_path, "config/upload_sample.ini"), mode="w", encoding="utf-8") as file: file.write(os.linesep.join(self.jfr_paths)) + def __stop_recorder_by_root(self): + for target in self.pids_to_start_recording: + username = psutil.Process(int(target.pid)).username() + command = f"su - {username} -c '{self.jcmd_path} {target.pid} JFR.stop name={self.RECORD_NAME}'" + outcome = shell_tools.exec_shell(command, is_shell=True) + logging.info(outcome) + def __stop_recorder(self): for target in self.pids_to_start_recording: outcome = shell_tools.exec_shell("jcmd {} JFR.stop name={}".format(target.pid, self.RECORD_NAME), is_shell=True) logging.info(outcome) + def __check_has_stopped_recorder_by_root(self): + for target in self.pids_to_start_recording: + username = psutil.Process(int(target.pid)).username() + command = f"su - {username} -c '{self.jcmd_path} {target.pid} JFR.check name={self.RECORD_NAME}'" + outcome = shell_tools.exec_shell(command, is_shell=True) + logging.info(outcome) + if outcome.out.find("Could not find"): + self.pids_to_stop_recording.append(target) + if len(self.pids_to_stop_recording) == len(self.pids_to_start_recording): + return True + else: + self.pids_to_stop_recording.clear() + return False + def __check_has_stopped_recorder(self): for target in self.pids_to_start_recording: outcome = shell_tools.exec_shell("jcmd {} JFR.check name={}".format(target.pid, self.RECORD_NAME), @@ -172,7 +247,7 @@ def main(): help="the task id of the sample") parser.add_argument("-w", "--wait-for-jmeter-stop", dest="waiting", action="store_true", help="the sample stop when the jmeter stop") - parser.set_defaults(root_path=obtain_root_path(ROOT_PATH)) + parser.set_defaults(root_path=PyInstallerUtils.obtain_root_path(ROOT_PATH)) args = parser.parse_args() config_log_ini(args.root_path, "devkit_tester_agent") logging.info("start") diff --git a/test/devkit_utils/__init__.py b/test/devkit_utils/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/test/devkit_utils/test_devkit_client.py b/test/devkit_utils/test_devkit_client.py new file mode 100644 index 0000000000000000000000000000000000000000..8d5788ee5805365ee0ac10d7ee0b4fb7aa67477b --- /dev/null +++ b/test/devkit_utils/test_devkit_client.py @@ -0,0 +1,37 @@ +import unittest +from unittest import TestCase + +from devkit_utils.devkit_client import DevKitClient + + +class TestDevKitClient(TestCase): + + @unittest.skip("跳过这个测试方法") + def test_login(self): + try: + client = DevKitClient("172.39.173.2", "8086", "devadmin", "Huawei12#$") + client.upload_report_by_force("/home/panlonglong/Downloads/Main(136462)") + client.logout() + except Exception as e: + print(str(e)) + self.fail() + + @unittest.skip("跳过这个测试方法") + def test_login_use_proxy(self): + try: + client = DevKitClient("172.39.173.2", "8086", "devadmin", "Huawei12#$") + client.use_proxy = False + client.upload_report_by_force("/home/panlonglong/Downloads/Main(136462)") + client.logout() + except Exception as e: + print(str(e)) + self.fail() + + def test_try_to_login(self): + self.fail() + + def test_logout(self): + self.fail() + + def test_upload_report_by_force(self): + self.fail()