From 3eb27dde0d2ebc1ddb26664e60ab1a34879e65d6 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 11 May 2022 15:36:48 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=97=A8=E7=A6=81=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E7=BC=BA=E9=99=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ac/acl/binary/check_binary_file.py | 49 ++++++++++--------- src/tools/obs_package_build_history.py | 44 ++++++++++------- src/tools/obs_package_build_report.py | 9 ++-- src/utils/check_abi.py | 32 ++++++------- src/utils/check_conf.py | 65 ++++++++++++++------------ 5 files changed, 108 insertions(+), 91 deletions(-) diff --git a/src/ac/acl/binary/check_binary_file.py b/src/ac/acl/binary/check_binary_file.py index c1d448e..506361c 100644 --- a/src/ac/acl/binary/check_binary_file.py +++ b/src/ac/acl/binary/check_binary_file.py @@ -18,10 +18,11 @@ import os import shutil import logging +from pyrpm.spec import Spec, replace_macros + from src.ac.framework.ac_base import BaseCheck from src.ac.framework.ac_result import FAILED, SUCCESS from src.ac.common.gitee_repo import GiteeRepo -from pyrpm.spec import Spec, replace_macros logger = logging.getLogger("ac") @@ -55,30 +56,6 @@ class CheckBinaryFile(BaseCheck): finally: shutil.rmtree(self._work_tar_dir) - def check_compressed_file(self): - """ - 解压缩包 - """ - need_compress_files = [] - for decompress_file in self._gr.get_compress_files(): - if decompress_file not in self._tarball_in_spec: - need_compress_files.append(decompress_file) - self._gr.set_compress_files(need_compress_files) - return SUCCESS if 0 == self._gr.decompress_all() else FAILED - - def check_binary(self): - """ - 检查二进制文件 - """ - suffixes_list = self._get_all_file_suffixes(self._work_tar_dir) - if suffixes_list: - logger_con = ["%s: \n%s" % (key, value) for suffix_list in suffixes_list for key, value in - suffix_list.items()] - logger.error("binary file of type exists:\n%s", "\n".join(logger_con)) - return FAILED - else: - return SUCCESS - @staticmethod def _is_compress_tar_file(filename): """ @@ -122,4 +99,26 @@ class CheckBinaryFile(BaseCheck): binary_list.append({root.split("code")[-1]: binary_file}) return binary_list + def check_compressed_file(self): + """ + 解压缩包 + """ + need_compress_files = [] + for decompress_file in self._gr.get_compress_files(): + if decompress_file not in self._tarball_in_spec: + need_compress_files.append(decompress_file) + self._gr.set_compress_files(need_compress_files) + return SUCCESS if 0 == self._gr.decompress_all() else FAILED + def check_binary(self): + """ + 检查二进制文件 + """ + suffixes_list = self._get_all_file_suffixes(self._work_tar_dir) + if suffixes_list: + logger_con = ["%s: \n%s" % (key, value) for suffix_list in suffixes_list + for key, value in suffix_list.items()] + logger.error("binary file of type exists:\n%s", "\n".join(logger_con)) + return FAILED + else: + return SUCCESS diff --git a/src/tools/obs_package_build_history.py b/src/tools/obs_package_build_history.py index fc4f231..5330f3d 100644 --- a/src/tools/obs_package_build_history.py +++ b/src/tools/obs_package_build_history.py @@ -15,18 +15,15 @@ # Description: report obs package build info # ********************************************************************************** """ - import gevent -from gevent import monkey -monkey.patch_all() -import logging.config +gevent.monkey.patch_all() import logging import os import argparse import time +import json import xml.etree.ElementTree as ET from xml.etree.ElementTree import ParseError -import json logger = logging.getLogger("common") @@ -52,10 +49,16 @@ class JobBuildHistory(object): duration = [int(ele.get("duration")) for ele in root.findall("entry")] if not duration: - return {"package": package, "max": 0, "min": 0, "average": 0, "times": 0, "duration": []} - - return {"package": package, "max": max(duration), "min": min(duration), "duration": duration, - "average": sum(duration) / len(duration), "times": len(duration)} + return {"package": package, "max": 0, "min": 0, "average": 0, "times": 0, + "duration": []} + build_time_info = {} + try: + build_time_info = {"package": package, "max": max(duration), "min": min(duration), + "duration": duration, + "average": sum(duration) / len(duration), "times": len(duration)} + except ZeroDivisionError: + logger.error("divide is 0") + return build_time_info @staticmethod def get_packages_job_duration(project, repo, arch, concurrency, *packages): @@ -68,12 +71,16 @@ class JobBuildHistory(object): :param concurrency: :return: """ - batch = (len(packages) + concurrency - 1) / concurrency + if concurrency != 0: + batch = (len(packages) + concurrency - 1) / concurrency + else: + logger.error("divide is 0") rs = [] for index in range(batch): - works = [gevent.spawn(JobBuildHistory.get_package_job_duration, project, package, repo, arch) - for package in packages[index * concurrency: (index + 1) * concurrency]] + works = [ + gevent.spawn(JobBuildHistory.get_package_job_duration, project, package, repo, arch) + for package in packages[index * concurrency: (index + 1) * concurrency]] logger.info("%s works, %s/%s ", len(works), index + 1, batch) gevent.joinall(works) for work in works: @@ -97,7 +104,8 @@ class JobBuildHistory(object): """ packages = OBSProxy.list_project(project) - return JobBuildHistory.get_packages_job_duration(project, repo, arch, concurrency, *packages) + return JobBuildHistory.get_packages_job_duration(project, repo, arch, concurrency, + *packages) if "__main__" == __name__: @@ -113,17 +121,19 @@ if "__main__" == __name__: args = args.parse_args() _ = not os.path.exists("log") and os.mkdir("log") - logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) + logger_conf_path = os.path.realpath( + os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") from src.proxy.obs_proxy import OBSProxy if args.packages: - result = JobBuildHistory.get_packages_job_duration(args.project, args.repo, args.arch, - int(args.concurrency), *args.packages) + result = JobBuildHistory.get_packages_job_duration(args.project, args.repo, args.arch, + int(args.concurrency), *args.packages) else: - result = JobBuildHistory.get_jobs_duration(args.project, args.repo, args.arch, int(args.concurrency)) + result = JobBuildHistory.get_jobs_duration(args.project, args.repo, args.arch, + int(args.concurrency)) if args.output: with open(args.output, "w") as f: diff --git a/src/tools/obs_package_build_report.py b/src/tools/obs_package_build_report.py index 70e5471..80323e3 100644 --- a/src/tools/obs_package_build_report.py +++ b/src/tools/obs_package_build_report.py @@ -85,11 +85,13 @@ class ObsPackageBuildReport(object): 导出 :return: """ - print("{:<25}{:<25}{:<15}{:<20}{:<20}".format("project", "package", "state", "gitee", "charger")) + line = "{:<25}{:<25}{:<15}{:<20}{:<20}".format("project", "package", "state", "gitee", "charger") + no_fmt_logger(line) for package in self._package_last_committer: committer = self._package_last_committer[package]["committer"] real_name = self._package_last_committer[package]["real_name"].encode("utf-8") - print("{:<25}{:<25}{:<15}{:<20}{:<20}".format(self._project, package, self._state, committer, real_name)) + line = "{:<25}{:<25}{:<15}{:<20}{:<20}".format(self._project, package, self._state, committer, real_name) + no_fmt_logger(line) if "__main__" == __name__: @@ -102,10 +104,11 @@ if "__main__" == __name__: args = args.parse_args() - not os.path.exists("log") and os.mkdir("log") + _ = not os.path.exists("log") and os.mkdir("log") logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") + no_fmt_logger = logging.getLogger("no_fmt") from src.proxy.obs_proxy import OBSProxy from src.proxy.gitee_proxy import GiteeProxy diff --git a/src/utils/check_abi.py b/src/utils/check_abi.py index ef8b934..7d182ee 100755 --- a/src/utils/check_abi.py +++ b/src/utils/check_abi.py @@ -1,4 +1,5 @@ #!/usr/bin/python3 +""" #****************************************************************************** # Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. # [openeuler-jenkins] is licensed under the Mulan PSL v2. @@ -11,22 +12,20 @@ # See the Mulan PSL v2 for more details. # Author: wangchuangGG # Create: 2020-07-20 +# Description: +# (1) This script is used to check the ABI changes between the old +# and new versions of dynamic libraries. +# The merged result on difference is saved in the xxx_all_abidiff.out file in the working +# directory. +# default path: /var/tmp/xxx_all_abidiff.md +# +# (2) This script depends on abidiff from libabigail package. +# +# (3) Command parameters +# This script accept three kinds of command: compare_rpm or compare_so or compare_rpms +# Run it without any paramter prints out help message. # ******************************************************************************/ - -""" -(1) This script is used to check the ABI changes between the old - and new versions of dynamic libraries. - The merged result on difference is saved in the xxx_all_abidiff.out file in the working - directory. - default path: /var/tmp/xxx_all_abidiff.md - -(2) This script depends on abidiff from libabigail package. - -(3) Command parameters - This script accept three kinds of command: compare_rpm or compare_so or compare_rpms - Run it without any paramter prints out help message. """ - import argparse import subprocess import sys @@ -36,6 +35,7 @@ import shutil import tempfile import re + class CheckAbi(object): """check abi functions""" @@ -49,7 +49,7 @@ class CheckAbi(object): self.target_sos = set() self.changed_sos = set() self.diff_result_file = "" - + def list_so_files(self, path, add_global): """ Generate a list of all .so files in the directory. @@ -216,7 +216,7 @@ class CheckAbi(object): logging.info("result write in: %s, returncode:%d", abidiff_file, ret.returncode) return_code |= ret.returncode logging.info("-----final return_code:%s", return_code) - if not return_code in [0, 1]: + if return_code not in [0, 1]: self.diff_result_file = self.merge_all_abidiff_files(all_abidiff_files, base_name) logging.info("abidiff all results writed in: %s", self.diff_result_file) return return_code diff --git a/src/utils/check_conf.py b/src/utils/check_conf.py index 9c0ee2c..b17d2e5 100755 --- a/src/utils/check_conf.py +++ b/src/utils/check_conf.py @@ -16,12 +16,13 @@ # ********************************************************************************** """ import os -import sys import argparse import logging import shutil import tempfile -import subprocess + +from src.utils.shell_cmd import shell_cmd + class CheckConfig(object): """check config file""" @@ -65,7 +66,7 @@ class CheckConfig(object): elif len(a_b) != 1: understand_configs = understand_configs + " in new file:" + line + "\n" new_conf.close() - if len(understand_configs): + if understand_configs: self._not_understand_configs[name] = understand_configs logging.debug("\n---not_understand_configs:%s----", self._not_understand_configs) @@ -73,22 +74,24 @@ class CheckConfig(object): add_configs = "" del_configs = "" for key in new_dict.keys(): # 修改的内容 - if old_dict.get(key) is not None: - if old_dict[key] != new_dict[key]: + old_value = old_dict.get(key) + new_value = new_dict.get(key) + if old_value is not None: + if old_value != new_value: changed_configs = changed_configs + "Key:" + key + " Old_value:" +\ - old_dict[key] + " New_value:" + new_dict[key] + "\n" + old_value + " New_value:" + new_value + "\n" old_dict.pop(key) else: - add_configs = add_configs + "Key:" + key + " Value:" + new_dict[key] + "\n" - if len(changed_configs): + add_configs = add_configs + "Key:" + key + " Value:" + new_value + "\n" + if changed_configs: self._changed_configs[name] = changed_configs logging.debug("\n---changed_configs:%s----", self._changed_configs) - if len(add_configs): + if add_configs: self._add_configs[name] = add_configs logging.debug("\n---add_configs:%s----", self._add_configs) for key in old_dict: # 删除的内容 del_configs = del_configs + "Key:" + key + " Value:" + old_dict[key] + "\n" - if len(del_configs): + if del_configs: self._delete_configs[name] = del_configs logging.debug("\n---delete_configs:%s----", self._delete_configs) @@ -96,8 +99,10 @@ class CheckConfig(object): """ Check md5sum """ - old_md5 = subprocess.run(['md5sum', old_rpm], stdout=subprocess.PIPE) - new_md5 = subprocess.run(['md5sum', new_rpm], stdout=subprocess.PIPE) + old_cmd = "md5sum {}".format(old_rpm) + _, old_md5, _ = shell_cmd(old_cmd) + new_cmd = "md5sum {}".format(new_rpm) + _, new_md5, _ = shell_cmd(new_cmd) return old_md5.stdout.split()[0] == new_md5.stdout.split()[0] def _check_diff(self, old_and_new_path): @@ -121,8 +126,8 @@ class CheckConfig(object): old_and_new_path = [os.path.join(temp_path, x) for x in ["old", "new"]] _ = [os.makedirs(x) for x in old_and_new_path] rpms = [self._old_rpm, self._new_rpm] - _ = [subprocess.run('cd {}; rpm2cpio {} | cpio -di > /dev/null 2>&1'.format(x[0], x[1]), - shell=True) for x in zip(old_and_new_path, rpms)] + _ = [shell_cmd('cd {}; rpm2cpio {} | cpio -di > /dev/null 2>&1'.format(x[0], x[1]))\ + for x in zip(old_and_new_path, rpms)] logging.debug("\n---old version path:%s new version path:%s----", old_and_new_path[0], old_and_new_path[1]) return old_and_new_path @@ -165,20 +170,6 @@ class CheckConfig(object): logging.info("\n---Configs are same----") os.remove(self._output_file) ofile.close() - - def _get_rpms(self, rpm_url, dest): - """ - Get rpm path - """ - rpm_path = "" - if os.path.isfile(rpm_url): - rpm_path = os.path.abspath(rpm_url) - else: - rpm_name = os.path.basename(rpm_url) - rpm_path = os.path.join(dest, rpm_name) - logging.info("downloading %s ...", rpm_name) - subprocess.run("wget -t 5 -c -P {} {}".format(dest, rpm_url), shell=True) - return rpm_path def conf_check(self): """ @@ -191,8 +182,8 @@ class CheckConfig(object): if self._md5_check(self._old_rpm, self._new_rpm): logging.info("Same RPM") return - old_config = subprocess.run(['rpm', '-qpc', self._old_rpm], stdout=subprocess.PIPE) - new_config = subprocess.run(['rpm', '-qpc', self._new_rpm], stdout=subprocess.PIPE) + _, old_config, _ = shell_cmd('rpm -qpc {}'.format(self._old_rpm)) + _, new_config, _ = shell_cmd('rpm -qpc {}'.format(self._new_rpm)) for line in old_config.stdout.split(): self._remove_file.add(line) for line in new_config.stdout.split(): @@ -210,6 +201,20 @@ class CheckConfig(object): logging.error("file not found") shutil.rmtree(temp_path) + def _get_rpms(self, rpm_url, dest): + """ + Get rpm path + """ + rpm_path = "" + if os.path.isfile(rpm_url): + rpm_path = os.path.abspath(rpm_url) + else: + rpm_name = os.path.basename(rpm_url) + rpm_path = os.path.join(dest, rpm_name) + logging.info("downloading %s ...", rpm_name) + shell_cmd("wget -t 5 -c -P {} {}".format(dest, rpm_url)) + return rpm_path + def parse_command_line(): """Parse the command line args""" -- Gitee