From 8ed992850ab889a87890e9c1a7d84c5f942af53e Mon Sep 17 00:00:00 2001 From: zoupanpan Date: Tue, 18 Jun 2024 11:38:47 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9docker=E7=9A=84?= =?UTF-8?q?=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/devkit_utils/docker_utils.py | 40 +++++++++++++++++++ .../bin/flight_records_sample.py | 40 +++++++++++++++++-- 2 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 common/devkit_utils/docker_utils.py diff --git a/common/devkit_utils/docker_utils.py b/common/devkit_utils/docker_utils.py new file mode 100644 index 0000000..1deabd7 --- /dev/null +++ b/common/devkit_utils/docker_utils.py @@ -0,0 +1,40 @@ +import logging +import os.path + +from devkit_utils import shell_tools + + +def get_docker_id(docker_path: str): + paths_name = docker_path.split("/") + if len(paths_name) >= 2: + return paths_name[1] + else: + raise Exception("can not found docker id") + + +def is_docker_process(pid): + cgroup_file = f'/proc/{pid}/cgroup' + if not os.path.exists(pid): + return False, None + with open(cgroup_file, "r", encoding="utf-8") as file: + cgroup_infos = file.readlines() + for line in cgroup_infos: + fields = line.strip(":") + if len(fields) == 3 and str(fields[1]) == "devices" and str(fields[2]).startswith("/docker"): + return True, get_docker_id(fields[2]) + return False, None + + +def create_dir_in_docker(docker_id, target_dir, mode=755): + outcome = shell_tools.exec_shell(f"docker exec {docker_id} mkdir -p {target_dir}", is_shell=True) + logging.info(outcome) + outcome = shell_tools.exec_shell(f"docker exec {docker_id} chmod {mode} {target_dir}", is_shell=True) + logging.info(outcome) + + +def copy_to_docker(docker_id, origin_file, target_dir): + file_name = os.path.basename(origin_file) + outcome = shell_tools.exec_shell(f"docker cp {origin_file} {docker_id}:{target_dir}", is_shell=True) + logging.info(outcome) + outcome = shell_tools.exec_shell(f"docker exec {docker_id} chmod 755 {target_dir}/{file_name}", is_shell=True) + logging.info(outcome) diff --git a/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py b/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py index 4d8355d..e6972e8 100644 --- a/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py +++ b/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py @@ -7,7 +7,7 @@ import time import psutil -from devkit_utils import shell_tools, file_utils +from devkit_utils import shell_tools, file_utils, docker_utils from devkit_utils.error_coce import ErrorCodeEnum from devkit_utils.log_config import config_log_ini from devkit_utils.pyinstaller_utils import PyInstallerUtils @@ -16,10 +16,13 @@ ROOT_PATH = os.path.dirname(os.path.dirname(__file__)) class TargetProcess: - def __init__(self, pid, name): + def __init__(self, pid, name, is_docker=False, docker_id=None): self.pid = pid self.name = name self.jfr_name = None + self.is_docker = is_docker + self.docker_id = docker_id + self.jfr_path = None class FlightRecordsFactory: @@ -47,6 +50,7 @@ class FlightRecordsFactory: file_utils.create_dir(self.dir_to_storage_jfr) self.jcmd_path = None self.user_is_root = False + self.is_docker = False def start_sample(self): try: @@ -82,6 +86,7 @@ class FlightRecordsFactory: logging.info("check has stopped recorder") while not self.__check_has_stopped_recorder_by_root() and (datetime.datetime.now() - before).seconds < 30: time.sleep(1) + self.__copy_to_host() else: logging.exception(f"The target application {self.apps} cannot be found or Operation not permitted") @@ -98,6 +103,7 @@ class FlightRecordsFactory: logging.info("check has stopped recorder") while not self.__check_has_stopped_recorder() and (datetime.datetime.now() - before).seconds < 30: time.sleep(1) + self.__copy_to_host() else: logging.exception(f"The target application {self.apps} cannot be found or Operation not permitted") @@ -126,7 +132,11 @@ class FlightRecordsFactory: logging.info("app:%s to pid %s", app, outcome) pids = outcome.out.split() for pid in pids: - self.pids.append(TargetProcess(pid, app)) + is_docker, docker_id = docker_utils.is_docker_process(pid) + if is_docker: + self.pids.append(TargetProcess(pid, app, is_docker, docker_id)) + else: + self.pids.append(TargetProcess(pid, app)) def __check_jcmd(self): commander_to_check = "which jcmd" @@ -148,6 +158,12 @@ class FlightRecordsFactory: logging.info(PyInstallerUtils.get_env()) logging.info(os.environ) for target in self.pids: + if target.is_docker: + docker_utils.create_dir_in_docker(target.docker_id, self.tmp_dir, mode=777) + docker_utils.create_dir_in_docker(target.docker_id, self.tmp_config_dir, mode=777) + docker_utils.create_dir_in_docker(target.docker_id, self.tmp_data_dir, mode=777) + docker_utils.copy_to_docker(target.docker_id, self.temporary_settings_path, + self.tmp_config_dir) jfr_path = self.__jfr_name(target.name, target.pid) username = psutil.Process(int(target.pid)).username() command = (f"su - {username} -c '" @@ -157,6 +173,7 @@ class FlightRecordsFactory: outcome = shell_tools.exec_shell(command, is_shell=True) logging.info(outcome) if outcome.return_code == 0: + target.jfr_path = jfr_path self.jfr_paths.append(jfr_path) self.pids_to_start_recording.append(target) # 移动到data目录下 @@ -165,6 +182,12 @@ class FlightRecordsFactory: def __start_recorder(self): for target in self.pids: + if target.is_docker: + docker_utils.create_dir_in_docker(target.docker_id, self.tmp_dir, mode=777) + docker_utils.create_dir_in_docker(target.docker_id, self.tmp_config_dir, mode=777) + docker_utils.create_dir_in_docker(target.docker_id, self.tmp_data_dir, mode=777) + docker_utils.copy_to_docker(target.docker_id, self.temporary_settings_path, + self.tmp_config_dir) jfr_path = self.__jfr_name(target.name, target.pid) command = (f"jcmd {target.pid} JFR.start settings={self.temporary_settings_path} duration={self.duration}s" f" name={self.RECORD_NAME} filename={jfr_path}") @@ -172,6 +195,7 @@ class FlightRecordsFactory: outcome = shell_tools.exec_shell(command, is_shell=True) logging.info(outcome) if outcome.return_code == 0: + target.jfr_path = jfr_path self.jfr_paths.append(jfr_path) self.pids_to_start_recording.append(target) # 移动到data目录下 @@ -218,6 +242,16 @@ class FlightRecordsFactory: self.pids_to_stop_recording.clear() return False + def __copy_to_host(self): + for target in self.pids_to_start_recording: + if target.is_docker: + outcome = shell_tools.exec_shell(f"docker cp {target.docker_id}:{target.jfr_path} {target.jfr_path}", + is_shell=True) + logging.info(outcome) + outcome = shell_tools.exec_shell(f"docker exec {target.docker_id} rm -rf {self.tmp_dir}", + is_shell=True) + logging.info(outcome) + def __jfr_name(self, app, pid): return os.path.join(self.tmp_data_dir, f"{app}_PID_{pid}_Time_{self.now_date}.jfr") -- Gitee From bd8b88dd56ab7ba5a9766f7305dfac3224d806cd Mon Sep 17 00:00:00 2001 From: zoupanpan Date: Tue, 18 Jun 2024 12:57:14 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=AF=B9docker=E7=9A=84?= =?UTF-8?q?=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/devkit_utils/docker_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/common/devkit_utils/docker_utils.py b/common/devkit_utils/docker_utils.py index 1deabd7..2c217c7 100644 --- a/common/devkit_utils/docker_utils.py +++ b/common/devkit_utils/docker_utils.py @@ -5,22 +5,22 @@ from devkit_utils import shell_tools def get_docker_id(docker_path: str): - paths_name = docker_path.split("/") - if len(paths_name) >= 2: - return paths_name[1] + paths_name = docker_path.strip().split("/") + if len(paths_name) >= 3: + return paths_name[2] else: raise Exception("can not found docker id") def is_docker_process(pid): cgroup_file = f'/proc/{pid}/cgroup' - if not os.path.exists(pid): + if not os.path.exists(cgroup_file): return False, None with open(cgroup_file, "r", encoding="utf-8") as file: cgroup_infos = file.readlines() for line in cgroup_infos: - fields = line.strip(":") - if len(fields) == 3 and str(fields[1]) == "devices" and str(fields[2]).startswith("/docker"): + fields = line.split(":") + if len(fields) >= 3 and str(fields[1]) == "devices" and str(fields[2]).startswith("/docker"): return True, get_docker_id(fields[2]) return False, None -- Gitee From b70c7ea08acfa209ebbeade67aaf50ce2fe34895 Mon Sep 17 00:00:00 2001 From: zoupanpan Date: Tue, 18 Jun 2024 13:02:24 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=86=97=E4=BD=99=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../devkit_tester_agent/bin/flight_records_sample.py | 1 - 1 file changed, 1 deletion(-) diff --git a/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py b/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py index e6972e8..40ee1d7 100644 --- a/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py +++ b/component/DevKitTester/devkit_tester_agent/bin/flight_records_sample.py @@ -156,7 +156,6 @@ class FlightRecordsFactory: def __start_recorder_by_root(self): logging.info(PyInstallerUtils.get_env()) - logging.info(os.environ) for target in self.pids: if target.is_docker: docker_utils.create_dir_in_docker(target.docker_id, self.tmp_dir, mode=777) -- Gitee From 502bbc3147c8dbf4de8ba9d64973db73a7cc1f17 Mon Sep 17 00:00:00 2001 From: zoupanpan Date: Tue, 18 Jun 2024 14:46:26 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A0=A1=E9=AA=8C?= =?UTF-8?q?=E5=8C=85=E5=A4=A7=E5=B0=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/download_and_deploy/src/download/download_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/download_and_deploy/src/download/download_config.py b/tools/download_and_deploy/src/download/download_config.py index 190ac25..57a3c1f 100644 --- a/tools/download_and_deploy/src/download/download_config.py +++ b/tools/download_and_deploy/src/download/download_config.py @@ -28,7 +28,7 @@ CompatibilityTesting = { DevKitTester = { "component_name": "DevKitTester", "file": "https://gitee.com/openeuler/devkit-pipeline/releases/download/v1.0.2-beta/devkit_tester.tar.gz", - "file_size": "36151864", + "file_size": "36153017", } A_FOT = { -- Gitee