From 815c425ca33a9f781154a0104a3f6d5dfa1590ca Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 27 Oct 2021 11:28:31 +0800 Subject: [PATCH 01/42] modify --- src/build/extra_work.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index fbefcc8..7cd6add 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -91,6 +91,7 @@ class ExtraWork(object): :return: """ #get rpms + logging.info("compare package start") curr_rpm = self._rpm_package.main_package_local() last_rpm = self._rpm_package.last_main_package(package_arch, package_url) logger.debug("curr_rpm: {}".format(curr_rpm)) @@ -152,6 +153,7 @@ class ExtraWork(object): comments.append(comment) logger.debug("check abi comments: {}".format(comments)) + logging.info("compare package finish") try: with open(comment_file, "w") as f: yaml.safe_dump(comments, f) # list -- Gitee From 2a0a4d5fb6bb85ff584a9978875388fb68580dec Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 27 Oct 2021 14:38:51 +0800 Subject: [PATCH 02/42] ac.yaml add zhengyaohui --- src/ac/framework/ac.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 5cabb6f..85fce10 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -42,3 +42,28 @@ openeuler: module: openlibing.check_code entry: CheckCode allow_list: ["pkgship", "kunpengsecl", "release-tools"] +zhengyaohui: + spec: + hint: check_spec_file + module: spec.check_spec + entry: CheckSpec + ignored: ["homepage"] + code: + hint: check_code_style + module: code.check_code_style + entry: CheckCodeStyle + #exclude: True + ignored: ["patch"] + package_yaml: + hint: check_package_yaml_file + module: package_yaml.check_yaml + entry: CheckPackageYaml + ignored: ["fields"] + package_license: + hint: check_package_license + module: package_license.check_license + entry: CheckLicense + sca: + exclude: True + openlibing: + exclude: True -- Gitee From 675876fa93892c2777167a4a6f9bac67e266c576 Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Sun, 31 Oct 2021 19:09:23 +0800 Subject: [PATCH 03/42] add compare package test --- src/build/extra_work.py | 67 +++++++++++++++++++++++++++++++++++++- src/build/gitee_comment.py | 48 ++++++++++++++++++++++++++- 2 files changed, 113 insertions(+), 2 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 7cd6add..fabafd0 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -160,6 +160,69 @@ class ExtraWork(object): except IOError: logger.exception("save check abi comment exception") + def compare_rpm_diff(self, package_url, package_arch, output, committer, comment_file, obs_addr, + branch_name="master", obs_repo_url=None): + """ + 对比两个版本rpm包之间的差异，根据差异找到受影响的rpm包 + :param package_arch: + :param obs_repo_url: + :return: + """ + #get rpms + logging.info("compare package start") + + logging.info("compare subpackage start:gcc-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-10.3.1-20210914.5.oe1.x86_64.rpm") + logging.info("compare abi pass") + logging.info("compare config pass") + logging.info("compare filelist pass") + logging.info("compare requires pass") + logging.info("compare provide pass") + logging.info("compare subpackage finish:gcc-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-10.3.1-20210914.5.oe1.x86_64.rpm") + + logging.info("compare subpackage start:gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm") + logging.info("compare abi pass") + logging.info("compare config pass") + logging.info("compare filelist pass") + logging.info("compare requires pass") + logging.info("compare provide pass") + logging.info("compare subpackage finish:gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm") + + logging.info("compare subpackage start:gcc-debuginfo-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-deebuginfo-10.3.1-20210914.5.oe1.x86_64.rpm") + logging.info("compare abi pass") + logging.info("compare config pass") + logging.info("compare filelist pass") + logging.info("compare requires pass") + logging.info("compare provide pass") + logging.info("compare subpackage finish:gcc-debuginfo-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-debuginfo-10.3.1-20210914.5.oe1.x86_64.rpm") + + comment = {"name": "compare_package/{}/{}".format(package_arch, self._repo), "result": "SUCCESS", + "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, + os.path.basename(output), package_url)} + + logger.debug("compare package comment: {}".format(comment)) + try: + with open(comment_file, "r") as f: # one repo with multi build package + comments = yaml.safe_load(f) + except IOError as e: + logger.debug("no history compare package comment") + + comments = [] + if os.path.exists(comment_file): + try: + with open(comment_file, "r") as f: # one repo with multi build package + comments = yaml.safe_load(f) + except IOError: + logger.exception("yaml load compare package comment file exception") + + comments.append(comment) + logger.debug("compare package comments: {}".format(comments)) + logging.info("compare package finish") + try: + with open(comment_file, "w") as f: + yaml.safe_dump(comments, f) # list + except IOError: + logger.exception("save compare package comment exception") + def check_install_rpm(self, branch_name, arch, install_root): """ 检查生成的rpm是否可以安装 @@ -254,7 +317,9 @@ if "__main__" == __name__: args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) elif args.func == "checkabi": # run before copy rpm to rpm repo - ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + #ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + #args.obs_addr, args.branch_name, args.obs_repo_url) + ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, args.obs_addr, args.branch_name, args.obs_repo_url) elif args.func == "checkinstall": ew.check_install_rpm(args.branch_name, args.arch, args.install_root) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 442ebbb..5d7e412 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -103,7 +103,8 @@ class Comment(object): comments.extend(self._comment_of_ac(self._up_up_builds[0])) if self._up_builds: comments.extend(self._comment_of_build(self._up_builds)) - comments.extend(self._comment_of_check_abi(self._up_builds)) + #comments.extend(self._comment_of_check_abi(self._up_builds)) + comments.extend(self._comment_of_compare_package(self._up_builds)) comments.append("") return comments @@ -162,6 +163,51 @@ class Comment(object): return comments + def _comment_of_compare_package(self, builds): + """ + compare package comment + :param builds: + :return: + """ + comments = [] + + def match(name, comment_file): + if "aarch64" in name and "aarch64" in comment_file: + return True + if "x86-64" in name and "x86_64" in comment_file: + return True + return False + + for check_abi_comment_file in self._check_abi_comment_files: + logger.debug("compare package comment file: {}".format(check_abi_comment_file)) + if not os.path.exists(check_abi_comment_file): # check abi评论文件存在 + continue + for build in builds: + name = build.job._data["fullName"] + logger.debug("check build {}".format(name)) + if not match(name, check_abi_comment_file): # 找到匹配的jenkins build + continue + logger.debug("build \"{}\" match".format(name)) + + status = build.get_status() + logger.debug("build state: {}".format(status)) + if ACResult.get_instance(status) == SUCCESS: # 保证build状态成功 + with open(check_abi_comment_file, "r") as f: + try: + content = yaml.safe_load(f) + except YAMLError: # yaml base exception + logger.exception("illegal yaml format of check abi comment file ") + logger.debug("comment: {}".format(content)) + for item in content: + ac_result = ACResult.get_instance(item.get("result")) + comments.append(self.__class__.comment_html_table_tr( + item.get("name"), ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) + break + + logger.info("compare package comment: {}".format(comments)) + + return comments + def _comment_of_check_abi(self, builds): """ check abi comment -- Gitee From 3328552eb925eb44dc632d69cbca8e213ba3dae5 Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Sun, 31 Oct 2021 11:14:05 +0000 Subject: [PATCH 04/42] update src/build/gitee_comment.py. --- src/build/gitee_comment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 5d7e412..3622b4f 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -201,7 +201,7 @@ class Comment(object): for item in content: ac_result = ACResult.get_instance(item.get("result")) comments.append(self.__class__.comment_html_table_tr( - item.get("name"), ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) + "compare_package", ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) break logger.info("compare package comment: {}".format(comments)) -- Gitee From 6003bf4d4a134a4df5390950d1709625b6a5bd40 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Fri, 5 Nov 2021 18:01:06 +0800 Subject: [PATCH 05/42] parse json file --- src/build/extra_work.py | 62 ++++++++++++++++++++++++----------------- src/build/test.py | 34 ++++++++++++++++++++++ 2 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 src/build/test.py diff --git a/src/build/extra_work.py b/src/build/extra_work.py index fabafd0..49614f8 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -21,6 +21,7 @@ import argparse import logging.config import logging import yaml +import json from src.build.obs_repo_source import OBSRepoSource @@ -160,8 +161,39 @@ class ExtraWork(object): except IOError: logger.exception("save check abi comment exception") + def output_result_to_console(result_file): + all_data = {} + with open(json_file, "r") as data: + all_data = json.loads(data) + + compare_result = all_data.get("compare result") + compare_details = all_data.get("compare details") + logger.info("compare package %s", compare_result) + if compare_details: + for key, value in compare_details.items(): + logger.info("%s:", key) + details = value.get("%s details" % key) + if key != "diff": + for rpm in details: + logger.info(rpm) + elif key == "diff": + diff_details = details + for rpm in details.keys(): + logger.info(rpm) + blank = " " + for key,value in diff_details.items(): + logger.info("compare %s not pass", key) + for key2,value2 in value.items(): + logger.info(blank + key2 + ":") + for key3,value3 in value2.items(): + logger.info(blank*2 + key3 + ":") + for key4, value4 in value3.items(): + logger.info(blank*3 + key4 + ":") + for line in value4: + logger.info(blank*4 + line) + def compare_rpm_diff(self, package_url, package_arch, output, committer, comment_file, obs_addr, - branch_name="master", obs_repo_url=None): + result_path, branch_name="master", obs_repo_url=None): """ 对比两个版本rpm包之间的差异，根据差异找到受影响的rpm包 :param package_arch: @@ -170,32 +202,10 @@ class ExtraWork(object): """ #get rpms logging.info("compare package start") + json_file = os.path.join(result_path, "result.json" + self.output_result_to_console(json_file) - logging.info("compare subpackage start:gcc-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-10.3.1-20210914.5.oe1.x86_64.rpm") - logging.info("compare abi pass") - logging.info("compare config pass") - logging.info("compare filelist pass") - logging.info("compare requires pass") - logging.info("compare provide pass") - logging.info("compare subpackage finish:gcc-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-10.3.1-20210914.5.oe1.x86_64.rpm") - - logging.info("compare subpackage start:gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm") - logging.info("compare abi pass") - logging.info("compare config pass") - logging.info("compare filelist pass") - logging.info("compare requires pass") - logging.info("compare provide pass") - logging.info("compare subpackage finish:gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm") - - logging.info("compare subpackage start:gcc-debuginfo-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-deebuginfo-10.3.1-20210914.5.oe1.x86_64.rpm") - logging.info("compare abi pass") - logging.info("compare config pass") - logging.info("compare filelist pass") - logging.info("compare requires pass") - logging.info("compare provide pass") - logging.info("compare subpackage finish:gcc-debuginfo-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-debuginfo-10.3.1-20210914.5.oe1.x86_64.rpm") - - comment = {"name": "compare_package/{}/{}".format(package_arch, self._repo), "result": "SUCCESS", + comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": "SUCCESS", "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, os.path.basename(output), package_url)} diff --git a/src/build/test.py b/src/build/test.py new file mode 100644 index 0000000..178bfa8 --- /dev/null +++ b/src/build/test.py @@ -0,0 +1,34 @@ +import json +def test(): + json_file = "/tmp/oecp/result2.json" + all_data = {} + blank = " " + with open(json_file, "r") as data: + all_data = json.load(data) + + compare_result = all_data.get("compare result") + compare_details = all_data.get("compare details") + print("compare package %s" %(compare_result)) + if compare_details: + for key, value in compare_details.items(): + print("%s:" % key) + details = value.get("%s details" % key) + if key != "diff": + for rpm in details: + print(rpm) + elif key == "diff": + diff_details = details + for rpm in details.keys(): + print(blank + rpm) + #for key,value in diff_details.items(): + # print("compare %s not pass" %(key)) + # for key2,value2 in value.items(): + # print(blank + key2) + # for key3,value3 in value2.items(): + # print(blank*2 + key3) + # for key4, value4 in value3.items(): + # print(blank*3 + key4) + # for line in value4: + # print(blank*4 + line) + +test() -- Gitee From dd8bec84e9758e948baefcc3347871d554a5d172 Mon Sep 17 00:00:00 2001 From: wanghuan158 Date: Mon, 8 Nov 2021 08:05:44 +0000 Subject: [PATCH 06/42] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=96=87=E4=BB=B6=20sr?= =?UTF-8?q?c/build/test.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/test.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 src/build/test.py diff --git a/src/build/test.py b/src/build/test.py deleted file mode 100644 index 178bfa8..0000000 --- a/src/build/test.py +++ /dev/null @@ -1,34 +0,0 @@ -import json -def test(): - json_file = "/tmp/oecp/result2.json" - all_data = {} - blank = " " - with open(json_file, "r") as data: - all_data = json.load(data) - - compare_result = all_data.get("compare result") - compare_details = all_data.get("compare details") - print("compare package %s" %(compare_result)) - if compare_details: - for key, value in compare_details.items(): - print("%s:" % key) - details = value.get("%s details" % key) - if key != "diff": - for rpm in details: - print(rpm) - elif key == "diff": - diff_details = details - for rpm in details.keys(): - print(blank + rpm) - #for key,value in diff_details.items(): - # print("compare %s not pass" %(key)) - # for key2,value2 in value.items(): - # print(blank + key2) - # for key3,value3 in value2.items(): - # print(blank*2 + key3) - # for key4, value4 in value3.items(): - # print(blank*3 + key4) - # for line in value4: - # print(blank*4 + line) - -test() -- Gitee From e45727a7cdbe42923745c0d2ef971e609a134a64 Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Mon, 8 Nov 2021 09:31:53 +0000 Subject: [PATCH 07/42] update src/build/extra_work.py. --- src/build/extra_work.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 49614f8..81d4607 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -304,6 +304,7 @@ if "__main__" == __name__: args.add_argument("-b", type=str, dest="obs_repo_url", help="obs repo where rpm saved") args.add_argument("-s", type=str, dest="obs_addr", help="obs address") args.add_argument("-r", type=str, dest="branch_name", help="obs project name") + args.add_argument("-j", type=str, dest="json_path", help="compare package json path") args.add_argument("--install-root", type=str, dest="install_root", help="check install root dir") args = args.parse_args() @@ -330,6 +331,6 @@ if "__main__" == __name__: #ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, #args.obs_addr, args.branch_name, args.obs_repo_url) ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, - args.obs_addr, args.branch_name, args.obs_repo_url) + args.obs_addr, args.json_path, args.branch_name, args.obs_repo_url) elif args.func == "checkinstall": ew.check_install_rpm(args.branch_name, args.arch, args.install_root) -- Gitee From 28e6b286cd09deaab29babe32f7cadfbe02a353b Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Mon, 8 Nov 2021 17:38:16 +0800 Subject: [PATCH 08/42] modify --- src/build/extra_work.py | 8 ++++---- src/build/test.py | 34 ---------------------------------- 2 files changed, 4 insertions(+), 38 deletions(-) delete mode 100644 src/build/test.py diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 49614f8..118b031 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -176,10 +176,10 @@ class ExtraWork(object): if key != "diff": for rpm in details: logger.info(rpm) - elif key == "diff": - diff_details = details - for rpm in details.keys(): - logger.info(rpm) + else: + diff_details = details + for rpm in details.keys(): + logger.info(rpm) blank = " " for key,value in diff_details.items(): logger.info("compare %s not pass", key) diff --git a/src/build/test.py b/src/build/test.py deleted file mode 100644 index 178bfa8..0000000 --- a/src/build/test.py +++ /dev/null @@ -1,34 +0,0 @@ -import json -def test(): - json_file = "/tmp/oecp/result2.json" - all_data = {} - blank = " " - with open(json_file, "r") as data: - all_data = json.load(data) - - compare_result = all_data.get("compare result") - compare_details = all_data.get("compare details") - print("compare package %s" %(compare_result)) - if compare_details: - for key, value in compare_details.items(): - print("%s:" % key) - details = value.get("%s details" % key) - if key != "diff": - for rpm in details: - print(rpm) - elif key == "diff": - diff_details = details - for rpm in details.keys(): - print(blank + rpm) - #for key,value in diff_details.items(): - # print("compare %s not pass" %(key)) - # for key2,value2 in value.items(): - # print(blank + key2) - # for key3,value3 in value2.items(): - # print(blank*2 + key3) - # for key4, value4 in value3.items(): - # print(blank*3 + key4) - # for line in value4: - # print(blank*4 + line) - -test() -- Gitee From 6c26b2bb36a21b9512d02c0bb0851cb9362591aa Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Tue, 9 Nov 2021 17:28:34 +0800 Subject: [PATCH 09/42] add result --- src/build/extra_work.py | 88 +++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 34b18b6..0923b22 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -25,6 +25,9 @@ import json from src.build.obs_repo_source import OBSRepoSource +SUCCESS = "SUCCESS" +FAILED = "FAILED" +DIFF_ITEM = ["more", "less", "diff"] class ExtraWork(object): """ @@ -160,37 +163,71 @@ class ExtraWork(object): yaml.safe_dump(comments, f) # list except IOError: logger.exception("save check abi comment exception") - - def output_result_to_console(result_file): + + def get_dict(self, key_list, data): + if not isinstance(data, dict): + return None + for key, value in data.items(): + if key == key_list[0]: + if len(key_list) > 1: + key_list = key_list[1:] + result = self.get_dict(key_list, value) + return result + else: + return value + + def get_item_data(self, item, data): + blank = " " + if isinstance(data, list): + logger.info("%s%s:", blank*2, item) + for item_data in data: + logger.info("%s%s", blank*3, item_data) + elif isinstance(data, dict): + logger.info("%s%s:", blank*2, item) + for key, value in data.items(): + logger.info("%s%s:", blank*3, key) + if isinstance(value, list): + for i in value: + logger.info("%s%s", blank*4, i) + else: + logger.info(value) + + def output_result_to_console(self, json_file): all_data = {} + if not os.path.exists(json_file): + logger.error("%s not exists", json_file) + return FAILED with open(json_file, "r") as data: - all_data = json.loads(data) + all_data = json.load(data) compare_result = all_data.get("compare result") compare_details = all_data.get("compare details") logger.info("compare package %s", compare_result) - if compare_details: - for key, value in compare_details.items(): - logger.info("%s:", key) - details = value.get("%s details" % key) - if key != "diff": - for rpm in details: - logger.info(rpm) - else: - diff_details = details - for rpm in details.keys(): - logger.info(rpm) + if not compare_details: + return FAILED + for key in DIFF_ITEM: + key_list = [key, "%s details" % key] + details = self.get_dict(key_list, compare_details) + if details: + logger.info("%s:", key) + for rpm in details: + logger.info(rpm) + diff_details = self.get_dict(["diff", "diff details"], compare_details) blank = " " for key,value in diff_details.items(): logger.info("compare %s not pass", key) - for key2,value2 in value.items(): - logger.info(blank + key2 + ":") - for key3,value3 in value2.items(): - logger.info(blank*2 + key3 + ":") - for key4, value4 in value3.items(): - logger.info(blank*3 + key4 + ":") - for line in value4: - logger.info(blank*4 + line) + compare_list = list(value.keys()) + for compare_item in compare_list: + logger.info(" %s:", compare_item) + item_list = [[compare_item, i] for i in DIFF_ITEM] + for items in item_list: + items_data = self.get_dict(items, value) + if items_data: + self.get_item_data(items[1], items_data) + if compare_result == "pass": + return SUCCESS + else: + return FAILED def compare_rpm_diff(self, package_url, package_arch, output, committer, comment_file, obs_addr, result_path, branch_name="master", obs_repo_url=None): @@ -202,10 +239,10 @@ class ExtraWork(object): """ #get rpms logging.info("compare package start") - json_file = os.path.join(result_path, "result.json" - self.output_result_to_console(json_file) + json_file = os.path.join(result_path, "result.json") + result = self.output_result_to_console(json_file) - comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": "SUCCESS", + comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": result, "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, os.path.basename(output), package_url)} @@ -334,3 +371,4 @@ if "__main__" == __name__: args.obs_addr, args.json_path, args.branch_name, args.obs_repo_url) elif args.func == "checkinstall": ew.check_install_rpm(args.branch_name, args.arch, args.install_root) + -- Gitee From b17868c8ae7fbf9c1bdacd5714fa550be2bc3fd4 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 10 Nov 2021 09:16:33 +0800 Subject: [PATCH 10/42] modify json file path --- src/build/extra_work.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 0923b22..de48ae4 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -239,8 +239,7 @@ class ExtraWork(object): """ #get rpms logging.info("compare package start") - json_file = os.path.join(result_path, "result.json") - result = self.output_result_to_console(json_file) + result = self.output_result_to_console(result_path) comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": result, "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, -- Gitee From b04139c35eb941618418d31380ab763865f05780 Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Thu, 11 Nov 2021 03:16:54 +0000 Subject: [PATCH 11/42] update src/build/extra_work.py. --- src/build/extra_work.py | 77 ++++++++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 32 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index de48ae4..5e1315e 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -229,8 +229,8 @@ class ExtraWork(object): else: return FAILED - def compare_rpm_diff(self, package_url, package_arch, output, committer, comment_file, obs_addr, - result_path, branch_name="master", obs_repo_url=None): + def compare_rpm_diff(self, package_url, package_arch, output, committer, comment_file, + result_path, pr_link): """ 对比两个版本rpm包之间的差异，根据差异找到受影响的rpm包 :param package_arch: @@ -317,33 +317,44 @@ class ExtraWork(object): if "__main__" == __name__: - args = argparse.ArgumentParser() - - args.add_argument("-f", type=str, dest="func", choices=("notify", "checkabi", "checkinstall"), help="function") - - args.add_argument("-p", type=str, dest="package", help="obs package") - args.add_argument("-a", type=str, dest="arch", help="build arch") - args.add_argument("-c", type=str, dest="committer", help="committer") - - args.add_argument("-d", type=str, dest="rpmbuild_dir", - default="/home/jenkins/agent/buildroot/home/abuild/rpmbuild", help="rpmbuild dir") - - args.add_argument("-n", type=str, dest="notify_url", help="target branch that merged to ") - args.add_argument("-t", type=str, dest="token", default=os.getcwd(), help="obs workspace dir path") - args.add_argument("-u", type=str, dest="notify_user", default="trigger", help="notify trigger user") - args.add_argument("-w", type=str, dest="notify_password", help="notify trigger password") - args.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") - args.add_argument("-m", type=str, dest="pkgship_meta", help="meta from pkgship spec") - - args.add_argument("-o", type=str, dest="output", help="checkabi result") - args.add_argument("-e", type=str, dest="comment_file", help="checkabi result comment") - args.add_argument("-b", type=str, dest="obs_repo_url", help="obs repo where rpm saved") - args.add_argument("-s", type=str, dest="obs_addr", help="obs address") - args.add_argument("-r", type=str, dest="branch_name", help="obs project name") - args.add_argument("-j", type=str, dest="json_path", help="compare package json path") - - args.add_argument("--install-root", type=str, dest="install_root", help="check install root dir") - args = args.parse_args() + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(help='sub-command help') + + #添加子命令 notify + parser_notify = subparsers.add_parser('notify', help='add help') + parser_notify.add_argument("-n", type=str, dest="notify_url", help="target branch that merged to ") + parser_notify.add_argument("-t", type=str, dest="token", default=os.getcwd(), help="obs workspace dir path") + parser_notify.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") + parser_notify.add_argument("-a", type=str, dest="arch", help="build arch") + parser_notify.add_argument("-u", type=str, dest="notify_user", default="trigger", help="notify trigger user") + parser_notify.add_argument("-w", type=str, dest="notify_password", help="notify trigger password") + + #添加子命令 checkabi + parser_checkabi = subparsers.add_parser('checkabi', help='add help') + parser_checkabi.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") + parser_checkabi.add_argument("-a", type=str, dest="arch", help="build arch") + parser_checkabi.add_argument("-o", type=str, dest="output", help="checkabi result") + parser_checkabi.add_argument("-c", type=str, dest="committer", help="committer") + parser_checkabi.add_argument("-s", type=str, dest="obs_addr", help="obs address") + parser_checkabi.add_argument("-r", type=str, dest="branch_name", help="obs project name") + parser_checkabi.add_argument("-b", type=str, dest="obs_repo_url", help="obs repo where rpm saved") + + #添加子命令 comparepackage + parser_comparepackage = subparsers.add_parser('comparepackage', help='add help') + parser_comparepackage.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") + parser_comparepackage.add_argument("-a", type=str, dest="arch", help="build arch") + parser_comparepackage.add_argument("-o", type=str, dest="output", help="checkabi result") + parser_comparepackage.add_argument("-c", type=str, dest="committer", help="committer") + parser_comparepackage.add_argument("-j", type=str, dest="json_path", help="compare package json path") + parser_comparepackage.add_argument("-p", type=str, dest="pr_link", help="PR link") + + #添加子命令 checkinstall + parser_checkinstall = subparsers.add_parser('checkinstall', help='add help') + parser_checkinstall.add_argument("-r", type=str, dest="branch_name", help="obs project name") + parser_checkinstall.add_argument("-a", type=str, dest="arch", help="build arch") + parser_checkinstall.add_argument("--install-root", type=str, dest="install_root", help="check install root dir") + + args = parser.parse_args() _ = not os.path.exists("log") and os.mkdir("log") logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) @@ -364,10 +375,12 @@ if "__main__" == __name__: args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) elif args.func == "checkabi": # run before copy rpm to rpm repo - #ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, - #args.obs_addr, args.branch_name, args.obs_repo_url) + ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + args.obs_addr, args.branch_name, args.obs_repo_url) + elif args.func == "comparepackage": + # run before copy rpm to rpm repo ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, - args.obs_addr, args.json_path, args.branch_name, args.obs_repo_url) + args.json_path, args.pr_link) elif args.func == "checkinstall": ew.check_install_rpm(args.branch_name, args.arch, args.install_root) -- Gitee From 927cdbf030a25af5eb30a358d52fba1ba4559d0f Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Thu, 11 Nov 2021 11:40:00 +0800 Subject: [PATCH 12/42] write pr_link to json --- src/build/extra_work.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 5e1315e..a40f6c5 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -192,14 +192,16 @@ class ExtraWork(object): else: logger.info(value) - def output_result_to_console(self, json_file): + def output_result_to_console(self, json_file, pr_link): all_data = {} if not os.path.exists(json_file): logger.error("%s not exists", json_file) return FAILED with open(json_file, "r") as data: all_data = json.load(data) - + with open(json_file, "w") as data: + all_data["pr link"] = pr_link + data.write(json.dump(all_data)) compare_result = all_data.get("compare result") compare_details = all_data.get("compare details") logger.info("compare package %s", compare_result) @@ -239,7 +241,7 @@ class ExtraWork(object): """ #get rpms logging.info("compare package start") - result = self.output_result_to_console(result_path) + result = self.output_result_to_console(result_path, pr_link) comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": result, "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, -- Gitee From 92b9954add5d38be8803ba2849b8b449b84e7de6 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Thu, 11 Nov 2021 14:14:16 +0800 Subject: [PATCH 13/42] delete repo print --- src/build/extra_work.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index a40f6c5..34e2028 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -204,7 +204,7 @@ class ExtraWork(object): data.write(json.dump(all_data)) compare_result = all_data.get("compare result") compare_details = all_data.get("compare details") - logger.info("compare package %s", compare_result) + logger.info("compare package %s", compare_result) if not compare_details: return FAILED for key in DIFF_ITEM: -- Gitee From 7957e9664652444ba8cc706f4f05768578f254af Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Thu, 11 Nov 2021 14:41:29 +0800 Subject: [PATCH 14/42] update args --- src/build/extra_work.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 34e2028..07a538c 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -320,6 +320,9 @@ class ExtraWork(object): if "__main__" == __name__: parser = argparse.ArgumentParser() + parser.add_argument("-p", type=str, dest="package", help="obs package") + parser.add_argument("-d", type=str, dest="rpmbuild_dir", + default="/home/jenkins/agent/buildroot/home/abuild/rpmbuild", help="rpmbuild dir") subparsers = parser.add_subparsers(help='sub-command help') #添加子命令 notify @@ -348,7 +351,7 @@ if "__main__" == __name__: parser_comparepackage.add_argument("-o", type=str, dest="output", help="checkabi result") parser_comparepackage.add_argument("-c", type=str, dest="committer", help="committer") parser_comparepackage.add_argument("-j", type=str, dest="json_path", help="compare package json path") - parser_comparepackage.add_argument("-p", type=str, dest="pr_link", help="PR link") + parser_comparepackage.add_argument("-pr", type=str, dest="pr_link", help="PR link") #添加子命令 checkinstall parser_checkinstall = subparsers.add_parser('checkinstall', help='add help') -- Gitee From 0be183b5932549d790b78adbabbeac4635edeef3 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Thu, 11 Nov 2021 15:12:48 +0800 Subject: [PATCH 15/42] update args --- src/build/extra_work.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 07a538c..a3f285f 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -333,6 +333,7 @@ if "__main__" == __name__: parser_notify.add_argument("-a", type=str, dest="arch", help="build arch") parser_notify.add_argument("-u", type=str, dest="notify_user", default="trigger", help="notify trigger user") parser_notify.add_argument("-w", type=str, dest="notify_password", help="notify trigger password") + parser_notify.set_defaults(func=notify) #添加子命令 checkabi parser_checkabi = subparsers.add_parser('checkabi', help='add help') @@ -343,6 +344,7 @@ if "__main__" == __name__: parser_checkabi.add_argument("-s", type=str, dest="obs_addr", help="obs address") parser_checkabi.add_argument("-r", type=str, dest="branch_name", help="obs project name") parser_checkabi.add_argument("-b", type=str, dest="obs_repo_url", help="obs repo where rpm saved") + parser_checkabi.set_defaults(func=checkabi) #添加子命令 comparepackage parser_comparepackage = subparsers.add_parser('comparepackage', help='add help') @@ -352,12 +354,14 @@ if "__main__" == __name__: parser_comparepackage.add_argument("-c", type=str, dest="committer", help="committer") parser_comparepackage.add_argument("-j", type=str, dest="json_path", help="compare package json path") parser_comparepackage.add_argument("-pr", type=str, dest="pr_link", help="PR link") + parser_comparepackage.set_defaults(func=comparepackage) #添加子命令 checkinstall parser_checkinstall = subparsers.add_parser('checkinstall', help='add help') parser_checkinstall.add_argument("-r", type=str, dest="branch_name", help="obs project name") parser_checkinstall.add_argument("-a", type=str, dest="arch", help="build arch") parser_checkinstall.add_argument("--install-root", type=str, dest="install_root", help="check install root dir") + parser_checkinstall.set_defaults(func=checkinstall) args = parser.parse_args() -- Gitee From 900d6a898676ddc805f168d9befea63e20b7a885 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Thu, 11 Nov 2021 15:31:32 +0800 Subject: [PATCH 16/42] update args --- src/build/extra_work.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index a3f285f..2cdbede 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -316,7 +316,23 @@ class ExtraWork(object): logger.error("install rpms error, {}, {}".format(ret, err)) else: logger.info("install rpm success") - + def notify(args, ew): + # run after copy rpm to rpm repo + if ew.is_pkgship_need_notify(args.pkgship_meta): + ew.pkgship_notify( + args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) + + def checkabi(args, ew): + # run before copy rpm to rpm repo + ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + args.obs_addr, args.branch_name, args.obs_repo_url) + + def comparepackage(args, ew): + ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + args.json_path, args.pr_link) + + def checkinstall(args, ew): + ew.check_install_rpm(args.branch_name, args.arch, args.install_root) if "__main__" == __name__: parser = argparse.ArgumentParser() @@ -377,19 +393,4 @@ if "__main__" == __name__: from src.utils.check_conf import CheckConfig ew = ExtraWork(args.package, args.rpmbuild_dir) - if args.func == "notify": - # run after copy rpm to rpm repo - if ew.is_pkgship_need_notify(args.pkgship_meta): - ew.pkgship_notify( - args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) - elif args.func == "checkabi": - # run before copy rpm to rpm repo - ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, - args.obs_addr, args.branch_name, args.obs_repo_url) - elif args.func == "comparepackage": - # run before copy rpm to rpm repo - ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, - args.json_path, args.pr_link) - elif args.func == "checkinstall": - ew.check_install_rpm(args.branch_name, args.arch, args.install_root) - + args.func(args, ew) -- Gitee From c3126cec451078b5d4dea7d59c0c880f09f89460 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Thu, 11 Nov 2021 16:36:46 +0800 Subject: [PATCH 17/42] update args --- src/build/extra_work.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 2cdbede..5b990d8 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -316,23 +316,24 @@ class ExtraWork(object): logger.error("install rpms error, {}, {}".format(ret, err)) else: logger.info("install rpm success") - def notify(args, ew): - # run after copy rpm to rpm repo - if ew.is_pkgship_need_notify(args.pkgship_meta): - ew.pkgship_notify( - args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) - - def checkabi(args, ew): - # run before copy rpm to rpm repo - ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, - args.obs_addr, args.branch_name, args.obs_repo_url) - - def comparepackage(args, ew): - ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, - args.json_path, args.pr_link) - - def checkinstall(args, ew): - ew.check_install_rpm(args.branch_name, args.arch, args.install_root) + +def notify(args, ew): + # run after copy rpm to rpm repo + if ew.is_pkgship_need_notify(args.pkgship_meta): + ew.pkgship_notify( + args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) + +def checkabi(args, ew): + # run before copy rpm to rpm repo + ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + args.obs_addr, args.branch_name, args.obs_repo_url) + +def comparepackage(args, ew): + ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + args.json_path, args.pr_link) + +def checkinstall(args, ew): + ew.check_install_rpm(args.branch_name, args.arch, args.install_root) if "__main__" == __name__: parser = argparse.ArgumentParser() -- Gitee From cbc78d17835ec679e154de16d656423339da4a83 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Thu, 11 Nov 2021 16:53:21 +0800 Subject: [PATCH 18/42] add args -e --- src/build/extra_work.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 5b990d8..d86ddb9 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -369,6 +369,7 @@ if "__main__" == __name__: parser_comparepackage.add_argument("-a", type=str, dest="arch", help="build arch") parser_comparepackage.add_argument("-o", type=str, dest="output", help="checkabi result") parser_comparepackage.add_argument("-c", type=str, dest="committer", help="committer") + parser_comparepackage.add_argument("-e", type=str, dest="comment_file", help="compare package result comment") parser_comparepackage.add_argument("-j", type=str, dest="json_path", help="compare package json path") parser_comparepackage.add_argument("-pr", type=str, dest="pr_link", help="PR link") parser_comparepackage.set_defaults(func=comparepackage) -- Gitee From e9ff64c126b40300f6457da8f1dfa4e39cc80fcf Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Thu, 11 Nov 2021 17:11:46 +0800 Subject: [PATCH 19/42] modify json.dump --- src/build/extra_work.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index d86ddb9..7d0f829 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -201,7 +201,7 @@ class ExtraWork(object): all_data = json.load(data) with open(json_file, "w") as data: all_data["pr link"] = pr_link - data.write(json.dump(all_data)) + json.dump(all_data, data) compare_result = all_data.get("compare result") compare_details = all_data.get("compare details") logger.info("compare package %s", compare_result) -- Gitee From b97b87cde39c44d02cbc8b29a6aa85c4904054b0 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Fri, 12 Nov 2021 09:22:10 +0800 Subject: [PATCH 20/42] print comment info --- src/build/extra_work.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 7d0f829..f258c9a 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -242,7 +242,11 @@ class ExtraWork(object): #get rpms logging.info("compare package start") result = self.output_result_to_console(result_path, pr_link) - + logger.info("committer = %s", committer) + logger.info("repo = %s", self._repo) + logger.info("package_arch = %s", package_arch) + logger.info("output = %s", os.path.basename(output)) + logger.info("package_url = %s", package_url) comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": result, "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, os.path.basename(output), package_url)} -- Gitee From 540ebbaae86d6c14e8a6cb6b60d5b515e75174a0 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Fri, 12 Nov 2021 09:54:35 +0800 Subject: [PATCH 21/42] delete print comment info --- src/build/extra_work.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index f258c9a..2ec95e8 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -204,7 +204,7 @@ class ExtraWork(object): json.dump(all_data, data) compare_result = all_data.get("compare result") compare_details = all_data.get("compare details") - logger.info("compare package %s", compare_result) + logger.info("compare <%s> package %s", self._repo, compare_result) if not compare_details: return FAILED for key in DIFF_ITEM: @@ -242,11 +242,6 @@ class ExtraWork(object): #get rpms logging.info("compare package start") result = self.output_result_to_console(result_path, pr_link) - logger.info("committer = %s", committer) - logger.info("repo = %s", self._repo) - logger.info("package_arch = %s", package_arch) - logger.info("output = %s", os.path.basename(output)) - logger.info("package_url = %s", package_url) comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": result, "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, os.path.basename(output), package_url)} -- Gitee From cbb40e5ce6f96ebe01475f0f51d26d7d6fadfe48 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Fri, 12 Nov 2021 10:48:12 +0800 Subject: [PATCH 22/42] add args --- src/build/extra_work.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 2ec95e8..4f7c13c 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -336,8 +336,8 @@ def checkinstall(args, ew): if "__main__" == __name__: parser = argparse.ArgumentParser() - parser.add_argument("-p", type=str, dest="package", help="obs package") - parser.add_argument("-d", type=str, dest="rpmbuild_dir", + parser.add_argument("-p", "--package", type=str, default="src-openeuler", help="obs package") + parser.add_argument("-d", "--rpmbuild_dir", type=str, default="/home/jenkins/agent/buildroot/home/abuild/rpmbuild", help="rpmbuild dir") subparsers = parser.add_subparsers(help='sub-command help') @@ -371,6 +371,7 @@ if "__main__" == __name__: parser_comparepackage.add_argument("-e", type=str, dest="comment_file", help="compare package result comment") parser_comparepackage.add_argument("-j", type=str, dest="json_path", help="compare package json path") parser_comparepackage.add_argument("-pr", type=str, dest="pr_link", help="PR link") + parser_comparepackage.add_argument("-p", "--package", type=str, help="obs package") parser_comparepackage.set_defaults(func=comparepackage) #添加子命令 checkinstall @@ -392,6 +393,6 @@ if "__main__" == __name__: from src.build.related_rpm_package import RelatedRpms from src.utils.check_abi import CheckAbi from src.utils.check_conf import CheckConfig - + ew = ExtraWork(args.package, args.rpmbuild_dir) args.func(args, ew) -- Gitee From 9fc1bff11411fa0460a9ee1c03ba547fbf52cdef Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Mon, 15 Nov 2021 14:47:13 +0800 Subject: [PATCH 23/42] add read file --- src/build/extra_work.py | 45 ++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 4f7c13c..8976b3b 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -176,23 +176,31 @@ class ExtraWork(object): else: return value - def get_item_data(self, item, data): + def get_item_data(self, item, data, fp): blank = " " + str_list = [] if isinstance(data, list): - logger.info("%s%s:", blank*2, item) + #logger.info("%s%s:", blank*2, item) + str_list.append(" {}:\n".format(item)) for item_data in data: - logger.info("%s%s", blank*3, item_data) + #logger.info("%s%s", blank*3, item_data) + str_list.append(" {}:\n".format(item_data)) elif isinstance(data, dict): - logger.info("%s%s:", blank*2, item) + #logger.info("%s%s:", blank*2, item) + str_list.append(" {}:\n".format(item)) for key, value in data.items(): - logger.info("%s%s:", blank*3, key) + #logger.info("%s%s:", blank*3, key) + str_list.append(" {}:\n".format(key)) if isinstance(value, list): for i in value: - logger.info("%s%s", blank*4, i) + #logger.info("%s%s", blank*4, i) + str_list.append(" {}:\n".format(i)) else: - logger.info(value) + #logger.info(value) + str_list.append("{}:\n".format(value)) + fp.writelines(str_list) - def output_result_to_console(self, json_file, pr_link): + def output_result_to_console(self, json_file, pr_link, fp): all_data = {} if not os.path.exists(json_file): logger.error("%s not exists", json_file) @@ -204,28 +212,32 @@ class ExtraWork(object): json.dump(all_data, data) compare_result = all_data.get("compare result") compare_details = all_data.get("compare details") - logger.info("compare <%s> package %s", self._repo, compare_result) + #logger.info("compare <%s> package %s", self._repo, compare_result) + fp.write("compare <%s> package %s\n" %(self._repo, compare_result)) if not compare_details: return FAILED for key in DIFF_ITEM: key_list = [key, "%s details" % key] details = self.get_dict(key_list, compare_details) if details: - logger.info("%s:", key) + #logger.info("%s:", key) + fp.write("%s:\n"%key) for rpm in details: logger.info(rpm) diff_details = self.get_dict(["diff", "diff details"], compare_details) blank = " " for key,value in diff_details.items(): - logger.info("compare %s not pass", key) + #logger.info("compare %s not pass", key) + fp.write("compare %s not pass\n" % key) compare_list = list(value.keys()) for compare_item in compare_list: - logger.info(" %s:", compare_item) + #logger.info(" %s:", compare_item) + fp.write("%s:\n" % compare_item) item_list = [[compare_item, i] for i in DIFF_ITEM] for items in item_list: items_data = self.get_dict(items, value) if items_data: - self.get_item_data(items[1], items_data) + self.get_item_data(items[1], items_data, fp) if compare_result == "pass": return SUCCESS else: @@ -241,7 +253,12 @@ class ExtraWork(object): """ #get rpms logging.info("compare package start") - result = self.output_result_to_console(result_path, pr_link) + tmp_file = "./tmp_result.txt" + with open(tmp_file, "w") af fp: + result = self.output_result_to_console(result_path, pr_link, fp) + with open(tmp_file, "r") af fp: + logger.info(fp.read()) + comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": result, "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, os.path.basename(output), package_url)} -- Gitee From 71a3d15344dca51e3d83d23ac356122900c135b4 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Mon, 15 Nov 2021 14:55:52 +0800 Subject: [PATCH 24/42] add read file --- src/build/extra_work.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 8976b3b..883a75e 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -254,9 +254,9 @@ class ExtraWork(object): #get rpms logging.info("compare package start") tmp_file = "./tmp_result.txt" - with open(tmp_file, "w") af fp: + with open(tmp_file, "w") as fp: result = self.output_result_to_console(result_path, pr_link, fp) - with open(tmp_file, "r") af fp: + with open(tmp_file, "r") as fp: logger.info(fp.read()) comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": result, -- Gitee From 271e07a93e888c04056d3c4b36f08f3ad3451f50 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Mon, 15 Nov 2021 15:11:17 +0800 Subject: [PATCH 25/42] modify print formet --- src/build/extra_work.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 883a75e..c8b23a9 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -177,27 +177,20 @@ class ExtraWork(object): return value def get_item_data(self, item, data, fp): - blank = " " str_list = [] if isinstance(data, list): - #logger.info("%s%s:", blank*2, item) str_list.append(" {}:\n".format(item)) for item_data in data: - #logger.info("%s%s", blank*3, item_data) - str_list.append(" {}:\n".format(item_data)) + str_list.append(" {}\n".format(item_data)) elif isinstance(data, dict): - #logger.info("%s%s:", blank*2, item) str_list.append(" {}:\n".format(item)) for key, value in data.items(): - #logger.info("%s%s:", blank*3, key) str_list.append(" {}:\n".format(key)) if isinstance(value, list): for i in value: - #logger.info("%s%s", blank*4, i) - str_list.append(" {}:\n".format(i)) + str_list.append(" {}\n".format(i)) else: - #logger.info(value) - str_list.append("{}:\n".format(value)) + str_list.append("{}\n".format(value)) fp.writelines(str_list) def output_result_to_console(self, json_file, pr_link, fp): @@ -212,7 +205,6 @@ class ExtraWork(object): json.dump(all_data, data) compare_result = all_data.get("compare result") compare_details = all_data.get("compare details") - #logger.info("compare <%s> package %s", self._repo, compare_result) fp.write("compare <%s> package %s\n" %(self._repo, compare_result)) if not compare_details: return FAILED @@ -220,18 +212,15 @@ class ExtraWork(object): key_list = [key, "%s details" % key] details = self.get_dict(key_list, compare_details) if details: - #logger.info("%s:", key) fp.write("%s:\n"%key) for rpm in details: logger.info(rpm) diff_details = self.get_dict(["diff", "diff details"], compare_details) blank = " " for key,value in diff_details.items(): - #logger.info("compare %s not pass", key) fp.write("compare %s not pass\n" % key) compare_list = list(value.keys()) for compare_item in compare_list: - #logger.info(" %s:", compare_item) fp.write("%s:\n" % compare_item) item_list = [[compare_item, i] for i in DIFF_ITEM] for items in item_list: -- Gitee From fb03cfa0edc634c015dd3c553d6484e518cc2a06 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Mon, 15 Nov 2021 15:16:33 +0800 Subject: [PATCH 26/42] modify print formet --- src/build/extra_work.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index c8b23a9..eadbb1a 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -212,9 +212,9 @@ class ExtraWork(object): key_list = [key, "%s details" % key] details = self.get_dict(key_list, compare_details) if details: - fp.write("%s:\n"%key) + fp.write("%s:\n" % key) for rpm in details: - logger.info(rpm) + fp.write("%s\n" % rpm) diff_details = self.get_dict(["diff", "diff details"], compare_details) blank = " " for key,value in diff_details.items(): -- Gitee From 120a2990af8d90adb4b2b51c3fdc4a04fbe96f37 Mon Sep 17 00:00:00 2001 From: wanghuan158 Date: Fri, 19 Nov 2021 10:52:54 +0000 Subject: [PATCH 27/42] =?UTF-8?q?update=20src/build/extra=5Fwork.py.=20?= =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=AD=97=E6=AE=B5=E5=90=8D=E7=A7=B0=E4=B8=8E?= =?UTF-8?q?oecp=E8=BE=93=E5=87=BA=E7=9A=84=E5=AD=97=E6=AE=B5=E5=8C=B9?= =?UTF-8?q?=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/extra_work.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index eadbb1a..d46c3a1 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -27,7 +27,6 @@ from src.build.obs_repo_source import OBSRepoSource SUCCESS = "SUCCESS" FAILED = "FAILED" -DIFF_ITEM = ["more", "less", "diff"] class ExtraWork(object): """ @@ -195,38 +194,41 @@ class ExtraWork(object): def output_result_to_console(self, json_file, pr_link, fp): all_data = {} + diff_item = ["more", "less", "diff"] + if not os.path.exists(json_file): logger.error("%s not exists", json_file) return FAILED with open(json_file, "r") as data: all_data = json.load(data) with open(json_file, "w") as data: - all_data["pr link"] = pr_link + all_data["pr_link"] = pr_link json.dump(all_data, data) - compare_result = all_data.get("compare result") - compare_details = all_data.get("compare details") + compare_result = all_data.get("compare_result") + compare_details = all_data.get("compare_details") fp.write("compare <%s> package %s\n" %(self._repo, compare_result)) if not compare_details: return FAILED - for key in DIFF_ITEM: - key_list = [key, "%s details" % key] + for key in diff_item: + key_list = [key, "%s_details" % key] details = self.get_dict(key_list, compare_details) if details: fp.write("%s:\n" % key) for rpm in details: fp.write("%s\n" % rpm) - diff_details = self.get_dict(["diff", "diff details"], compare_details) - blank = " " + + diff_details = self.get_dict(["diff", "diff_details"], compare_details) for key,value in diff_details.items(): fp.write("compare %s not pass\n" % key) compare_list = list(value.keys()) for compare_item in compare_list: fp.write("%s:\n" % compare_item) - item_list = [[compare_item, i] for i in DIFF_ITEM] + item_list = [[compare_item, i] for i in diff_item] for items in item_list: items_data = self.get_dict(items, value) if items_data: self.get_item_data(items[1], items_data, fp) + if compare_result == "pass": return SUCCESS else: -- Gitee From fd8e0258d8649c337af77b6f5974a2820073d9c5 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Mon, 22 Nov 2021 11:02:49 +0800 Subject: [PATCH 28/42] modify show formit --- src/build/extra_work.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index d46c3a1..4c06bf8 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -175,7 +175,8 @@ class ExtraWork(object): else: return value - def get_item_data(self, item, data, fp): + @staticmethod + def get_item_data(item, data, fp): str_list = [] if isinstance(data, list): str_list.append(" {}:\n".format(item)) @@ -192,6 +193,22 @@ class ExtraWork(object): str_list.append("{}\n".format(value)) fp.writelines(str_list) + def show_rpm_diff(self, compare_details, fp): + for key in ["more", "less", "diff"]: + key_list = [key, "%s_details" % key] + details = self.get_dict(key_list, compare_details) + if not details: + continue + if key == "diff": + fp.write("diff[old_rpm_name,new_rpm_name]:\n") + for rpm in details: + new_rpm_name = self.get_dict([rpm, "name", "new"], details) + fp.write("%s,%s\n" %(rpm, new_rpm_name)) + else: + fp.write("%s:\n" % key) + for rpm in details: + fp.write("%s\n" % rpm) + def output_result_to_console(self, json_file, pr_link, fp): all_data = {} diff_item = ["more", "less", "diff"] @@ -201,6 +218,7 @@ class ExtraWork(object): return FAILED with open(json_file, "r") as data: all_data = json.load(data) + # 写入pr link到json文件 with open(json_file, "w") as data: all_data["pr_link"] = pr_link json.dump(all_data, data) @@ -209,19 +227,18 @@ class ExtraWork(object): fp.write("compare <%s> package %s\n" %(self._repo, compare_result)) if not compare_details: return FAILED - for key in diff_item: - key_list = [key, "%s_details" % key] - details = self.get_dict(key_list, compare_details) - if details: - fp.write("%s:\n" % key) - for rpm in details: - fp.write("%s\n" % rpm) + # 显示所有的差异 + self.show_rpm_diff(compare_details, fp) + # 显示diff的详细差异 diff_details = self.get_dict(["diff", "diff_details"], compare_details) - for key,value in diff_details.items(): - fp.write("compare %s not pass\n" % key) + for key, value in diff_details.items(): + new_rpm_name = self.get_dict(["name", "new"], value) + fp.write("compare %s,%s not pass\n" % (key, new_rpm_name)) compare_list = list(value.keys()) for compare_item in compare_list: + if compare_item == "name": + continue fp.write("%s:\n" % compare_item) item_list = [[compare_item, i] for i in diff_item] for items in item_list: -- Gitee From f861279a727e4cbc7baef7a1b41df25c8c753f49 Mon Sep 17 00:00:00 2001 From: wanghuan158 Date: Mon, 22 Nov 2021 10:23:02 +0000 Subject: [PATCH 29/42] =?UTF-8?q?checkabi=E5=8F=82=E6=95=B0=E5=AE=8C?= =?UTF-8?q?=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/extra_work.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 4c06bf8..d8389f1 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -385,6 +385,8 @@ if "__main__" == __name__: parser_checkabi.add_argument("-s", type=str, dest="obs_addr", help="obs address") parser_checkabi.add_argument("-r", type=str, dest="branch_name", help="obs project name") parser_checkabi.add_argument("-b", type=str, dest="obs_repo_url", help="obs repo where rpm saved") + parser_checkabi.add_argument("-p", "--package", type=str, help="obs package") + parser_checkabi.add_argument("-e", type=str, dest="comment_file", help="compare package result comment") parser_checkabi.set_defaults(func=checkabi) #添加子命令 comparepackage -- Gitee From d1eb2162855e8f6325ded5a58f53aa80f9d77f0a Mon Sep 17 00:00:00 2001 From: wanghuan158 Date: Tue, 23 Nov 2021 01:16:40 +0000 Subject: [PATCH 30/42] =?UTF-8?q?comment=E4=BF=A1=E6=81=AF=E5=8A=A0?= =?UTF-8?q?=E4=B8=8Acheckabi?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/gitee_comment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 3622b4f..e6e94fe 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -103,7 +103,7 @@ class Comment(object): comments.extend(self._comment_of_ac(self._up_up_builds[0])) if self._up_builds: comments.extend(self._comment_of_build(self._up_builds)) - #comments.extend(self._comment_of_check_abi(self._up_builds)) + comments.extend(self._comment_of_check_abi(self._up_builds)) comments.extend(self._comment_of_compare_package(self._up_builds)) comments.append("") -- Gitee From d8bc1e6d4eafd7a7166ac1b571d18484eb08a3e6 Mon Sep 17 00:00:00 2001 From: wanghuan158 Date: Tue, 23 Nov 2021 01:17:34 +0000 Subject: [PATCH 31/42] =?UTF-8?q?=E5=88=A0=E9=99=A4checkabi=E9=87=8C?= =?UTF-8?q?=E9=9D=A2=E7=9A=84compare=20package=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/extra_work.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index d8389f1..b558474 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -94,7 +94,6 @@ class ExtraWork(object): :return: """ #get rpms - logging.info("compare package start") curr_rpm = self._rpm_package.main_package_local() last_rpm = self._rpm_package.last_main_package(package_arch, package_url) logger.debug("curr_rpm: {}".format(curr_rpm)) @@ -156,7 +155,6 @@ class ExtraWork(object): comments.append(comment) logger.debug("check abi comments: {}".format(comments)) - logging.info("compare package finish") try: with open(comment_file, "w") as f: yaml.safe_dump(comments, f) # list -- Gitee From 7630689560be328788ddbcb50bc9ef9a4fbf32cf Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Tue, 23 Nov 2021 10:17:00 +0800 Subject: [PATCH 32/42] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=AE=98=E6=96=B9?= =?UTF-8?q?=E4=BB=93=E5=BA=93=E4=BB=A3=E7=A0=81=E8=87=B3master=E5=88=86?= =?UTF-8?q?=E6=94=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README-zh-cn.md | 9 ++ README.md | 2 +- src/ac/acl/code/check_code_style.py | 18 ++-- src/ac/acl/openlibing/check_code.py | 49 +++++++++-- src/ac/acl/package_license/check_license.py | 13 +-- src/ac/acl/package_yaml/check_repo.py | 34 ++++---- src/ac/acl/package_yaml/check_yaml.py | 20 ++--- src/ac/acl/sca/check_sca.py | 70 ++++----------- src/ac/acl/spec/check_spec.py | 24 ++--- src/ac/common/gitee_repo.py | 18 ++-- src/ac/common/linter.py | 12 +-- src/ac/common/rpm_spec_adapter.py | 8 +- src/ac/common/scanoss.py | 10 +-- src/ac/framework/ac.py | 50 +++++------ src/ac/framework/ac.yaml | 25 ------ src/ac/framework/ac_base.py | 18 ++-- src/build/extra_work.py | 97 ++++----------------- src/build/gitee_comment.py | 68 +++------------ src/build/obs_repo_source.py | 12 +-- src/build/osc_build_k8s.py | 46 +++++----- src/jobs/jenkins_job.py | 18 ++-- src/jobs/obs_meta_strategy.py | 10 +-- src/jobs/repo_mapping.py | 10 +-- src/proxy/es_proxy.py | 12 +-- src/proxy/git_proxy.py | 26 +++--- src/proxy/gitee_proxy.py | 14 +-- src/proxy/jenkins_proxy.py | 28 +++--- src/proxy/kafka_proxy.py | 4 +- src/proxy/obs_proxy.py | 25 +++--- src/proxy/requests_proxy.py | 12 +-- src/requirements | 2 +- src/tools/obs_package_build_history.py | 8 +- src/tools/obs_package_build_report.py | 6 +- src/utils/shell_cmd.py | 16 ++-- test/ac/acl/package_yaml/test_check_repo.py | 4 +- 35 files changed, 338 insertions(+), 460 deletions(-) create mode 100644 README-zh-cn.md diff --git a/README-zh-cn.md b/README-zh-cn.md new file mode 100644 index 0000000..965092f --- /dev/null +++ b/README-zh-cn.md @@ -0,0 +1,9 @@ +# openeuler-jenkins + +## 简介 + +此仓库用来存放openEuler社区的Jenkins脚本。 + +## 许可证 + +详情请参考[LICENSE](https://gitee.com/openeuler/openeuler-jenkins/blob/ac397ce3e078937c700df6fb8de0e1b065ee4218/LICENSE)文件。 \ No newline at end of file diff --git a/README.md b/README.md index b9ed15e..70be875 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Introduction -This repository is used to store the jenkins scripts in openEuler Community. +This repository is used to store the Jenkins scripts in the openEuler community. ## License diff --git a/src/ac/acl/code/check_code_style.py b/src/ac/acl/code/check_code_style.py index cb574b5..7dde416 100644 --- a/src/ac/acl/code/check_code_style.py +++ b/src/ac/acl/code/check_code_style.py @@ -68,7 +68,7 @@ class CheckCodeStyle(BaseCheck): """ gp = GitProxy(self._work_dir) diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") - logger.debug("diff files: {}".format(diff_files)) + logger.debug("diff files: %s", diff_files) diff_code_files = [] # 仓库中变更的代码文件 diff_patch_code_files = [] # patch内的代码文件 @@ -77,7 +77,7 @@ class CheckCodeStyle(BaseCheck): diff_code_files.append(diff_file) elif GiteeRepo.is_patch_file(diff_file): patch_dir = self._gr.patch_dir_mapping.get(diff_file) - logger.debug("diff patch {} apply at dir {}".format(diff_file, patch_dir)) + logger.debug("diff patch %s apply at dir %s", diff_file, patch_dir) if patch_dir is not None: files_in_patch = gp.extract_files_path_of_patch(diff_file) patch_code_files = [os.path.join(patch_dir, file_in_patch) @@ -88,13 +88,13 @@ class CheckCodeStyle(BaseCheck): for code_file in patch_code_files if os.path.exists(code_file)]) - logger.debug("diff code files: {}".format(diff_code_files)) - logger.debug("diff patch code files: {}".format(diff_patch_code_files)) + logger.debug("diff code files: %s", diff_code_files) + logger.debug("diff patch code files: %s", diff_patch_code_files) rs_1 = self.check_file_under_work_dir(diff_code_files) - logger.debug("check_file_under_work_dir: {}".format(rs_1)) + logger.debug("check_file_under_work_dir: %s", rs_1) rs_2 = self.check_files_inner_patch(diff_patch_code_files) - logger.debug("check_files_inner_patch: {}".format(rs_2)) + logger.debug("check_files_inner_patch: %s", rs_2) return rs_1 + rs_2 @@ -131,10 +131,10 @@ class CheckCodeStyle(BaseCheck): elif GiteeRepo.is_c_cplusplus_file(file_path): rs = LinterCheck.check_c_cplusplus(file_path) else: - logger.error("error when arrive here, unsupport file {}".format(file_path)) + logger.error("error when arrive here, unsupport file %s", file_path) return SUCCESS - logger.info("Linter: {:<40} {}".format(file_path, rs)) + logger.info("Linter: %s %s", file_path, rs) if rs.get("F", 0) > 0: return FAILED @@ -150,7 +150,7 @@ class CheckCodeStyle(BaseCheck): :param kwargs: :return: """ - logger.info("check {} repo ...".format(self._repo)) + logger.info("check %s repo ...", self._repo) _ = not os.path.exists(self._work_tar_dir) and os.mkdir(self._work_tar_dir) try: diff --git a/src/ac/acl/openlibing/check_code.py b/src/ac/acl/openlibing/check_code.py index cd0ab58..d058509 100644 --- a/src/ac/acl/openlibing/check_code.py +++ b/src/ac/acl/openlibing/check_code.py @@ -17,6 +17,7 @@ """ import logging +import time from src.ac.framework.ac_base import BaseCheck from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS @@ -40,14 +41,45 @@ class CheckCode(BaseCheck): super(CheckCode, self).__init__(workspace, repo, conf) @staticmethod - def get_codecheck_result(pr_url, codecheck_api_url): + def get_codecheck_result(pr_url, codecheck_api_url, codecheck_api_key): """ 通过api调用codecheck """ - data = {"pr_url": pr_url} + # get codecheck Api Token + codecheck_token_api_url = '{}/token/{}'.format(codecheck_api_url, codecheck_api_key) + token_resp = {} + rs = do_requests("get", codecheck_token_api_url, obj=token_resp) + if rs != 0 or token_resp.get("code", "") != "200": + logger.error("get dynamic token failed") + return 'false', {} + + token = token_resp.get("data") + data = {"pr_url": pr_url, "token": token} response_content = {} + # 创建codecheck检查任务 + codecheck_task_api_url = "{}/task".format(codecheck_api_url) + rs = do_requests("get", codecheck_task_api_url, querystring=data, obj=response_content) + if rs != 0 or response_content.get('code', '') != '200': + logger.error("create codecheck task failed; %s", response_content.get('msg', '')) + return 'false', {} + + uuid = response_content.get('uuid') + task_id = response_content.get('task_id') + data = {"uuid": uuid, "token": token} + codecheck_status_api_url = '{}/{}/status'.format(codecheck_api_url, task_id) + current_time = 0 logger.info("codecheck probably need to 3min") - rs = do_requests("get", codecheck_api_url, querystring=data, timeout=180, obj=response_content) + # 定时3min + while current_time < 180: + time.sleep(10) + response_content = {} + # 检查codecheck任务的执行状态 + rs = do_requests("get", codecheck_status_api_url, querystring=data, obj=response_content) + if rs == 0 and response_content.get('code') == '100': + current_time = current_time + 10 + continue + else: + break return rs, response_content def check_code(self): @@ -55,7 +87,7 @@ class CheckCode(BaseCheck): 开始进行codecheck检查 """ # 等待计算结果 - rs, response_content = self.get_codecheck_result(self._pr_url, self._codecheck_api_url) + rs, response_content = self.get_codecheck_result(self._pr_url, self._codecheck_api_url, self._codecheck_api_key) # 判断是否计算完成 if rs != 0: @@ -70,12 +102,12 @@ class CheckCode(BaseCheck): "state": "pass(通过)/no pass(不通过)" } """ + logger.warning("click %s view code check detail", response_content.get('data')) # 只有codecheck完成且codecheck检查的代码中存在bug，返回检查项失败的结果，以detail结尾，会显示具体的代码bug所在位置。 if response_content.get("state") == "no pass": - logger.warning("click {} view code check detail".format(response_content.get('data'))) return FAILED else: - logger.error("code check failed, info :{}".format(response_content.get('msg'))) + logger.error("code check failed, info : %s", response_content.get('msg')) return SUCCESS @@ -86,12 +118,13 @@ class CheckCode(BaseCheck): :param kwargs: :return: """ - logger.info("check {} code ...".format(self._repo)) - logger.debug("args: {}, kwargs: {}".format(args, kwargs)) + logger.info("check %s code ...", self._repo) + logger.debug("args: %s, kwargs: %s", args, kwargs) codecheck_conf = kwargs.get("codecheck", {}) self._pr_url = codecheck_conf.get("pr_url", "") self._pr_number = codecheck_conf.get("pr_number", "") self._codecheck_api_url = codecheck_conf.get("codecheck_api_url", "") + self._codecheck_api_key = codecheck_conf.get('codecheck_api_key', "") return self.start_check() diff --git a/src/ac/acl/package_license/check_license.py b/src/ac/acl/package_license/check_license.py index 6502a02..669c139 100644 --- a/src/ac/acl/package_license/check_license.py +++ b/src/ac/acl/package_license/check_license.py @@ -87,13 +87,14 @@ class CheckLicense(BaseCheck): check whether the license in spec file and in src file is same :return """ - if self._pkg_license.check_licenses_is_same(self._license_in_spec, self._license_in_src, self._pkg_license._later_support_license): - logger.info("licenses in src:{} and in spec:{} are same".format(self._license_in_src, - self._license_in_spec)) + if self._pkg_license.check_licenses_is_same(self._license_in_spec, self._license_in_src, + self._pkg_license._later_support_license): + logger.info("licenses in src:%s and in spec:%s are same", self._license_in_src, + self._license_in_spec) return SUCCESS else: - logger.error("licenses in src:{} and in spec:{} are not same".format(self._license_in_src, - self._license_in_spec)) + logger.error("licenses in src:%s and in spec:%s are not same", self._license_in_src, + self._license_in_spec) return WARNING def __call__(self, *args, **kwargs): @@ -103,7 +104,7 @@ class CheckLicense(BaseCheck): :param kwargs: :return: """ - logger.info("check {} license ...".format(self._repo)) + logger.info("check %s license ...", self._repo) _ = not os.path.exists(self._work_tar_dir) and os.mkdir(self._work_tar_dir) self._gr.decompress_all() # decompress all compressed file into work_tar_dir diff --git a/src/ac/acl/package_yaml/check_repo.py b/src/ac/acl/package_yaml/check_repo.py index 839fd0b..11cdd96 100644 --- a/src/ac/acl/package_yaml/check_repo.py +++ b/src/ac/acl/package_yaml/check_repo.py @@ -72,7 +72,7 @@ class DefaultReleaseTags(AbsReleaseTags): 通过url获取上游社区的release tags return: list """ - logging.info("unsupported version control: {}".format(self.version_control)) + logging.info("unsupported version control: %s", self.version_control) return [] @@ -118,7 +118,7 @@ class HttpReleaseTagsMixin(object): response = requests.get(url, headers=headers, timeout=timeout) need_redirect, new_url, cookies = self.get_redirect_resp(url, response) if tldextract.extract(url).domain != tldextract.extract(new_url).domain: # 判断域名是否一致 预防csrf攻击 - logging.warning("domain of redirection link is different: {}".format(new_url)) + logging.warning("domain of redirection link is different: %s", new_url) return "" if need_redirect: cookie_dict = {} @@ -128,13 +128,13 @@ class HttpReleaseTagsMixin(object): url = new_url response = requests.get(url, headers=headers, cookies=cookie_dict, timeout=timeout) except requests.exceptions.SSLError as e: - logging.warning("requests {} ssl exception, {}".format(url, e)) + logging.warning("requests %s ssl exception, %s", url, e) return "" except requests.exceptions.Timeout as e: logging.warning("requests timeout") return "" except requests.exceptions.RequestException as e: - logging.warning("requests exception, {}".format(e)) + logging.warning("requests exception, %s", e) return "" return response @@ -156,7 +156,7 @@ class HgReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -170,7 +170,7 @@ class HgReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): temp_tags.sort(reverse=True, key=lambda x: x["date"][0]) release_tags = [tag["tag"] for tag in temp_tags] except Exception as e: - logging.error("exception, {}".format(e)) + logging.error("exception, %s", e) return [] return release_tags @@ -192,7 +192,7 @@ class HgRawReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -220,7 +220,7 @@ class MetacpanReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -258,7 +258,7 @@ class PypiReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -267,7 +267,7 @@ class PypiReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): tags_json = response.json() release_tags = [tag for tag in tags_json.get("releases")] except Exception as e: - logging.error("exception, {}".format(e)) + logging.error("exception, %s", e) return [] return release_tags @@ -289,7 +289,7 @@ class RubygemReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -301,7 +301,7 @@ class RubygemReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): if element.get("number"): release_tags.append(element.get("number")) except Exception as e: - logging.error("exception, {}".format(e)) + logging.error("exception, %s", e) return [] return release_tags @@ -323,7 +323,7 @@ class GnuftpReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -356,7 +356,7 @@ class FtpReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -382,7 +382,7 @@ class CmdReleaseTagsMixin(object): sub_proc = subprocess.Popen(cmd_list, stdout=subprocess.PIPE) response = sub_proc.stdout.read().decode("utf-8") if sub_proc.wait(): - logging.warning("{cmd} > encount errors".format(cmd=" ".join(cmd_list))) + logging.warning("%s > encount errors", " ".join(cmd_list)) return response @@ -411,7 +411,7 @@ class SvnReleaseTags(AbsReleaseTags, CmdReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get svn tags".format(repo=url)) + logging.debug("%s : get svn tags", url) if not url: logging.warning("illegal url: \"\"") return [] @@ -465,7 +465,7 @@ class GitReleaseTags(AbsReleaseTags, CmdReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] diff --git a/src/ac/acl/package_yaml/check_yaml.py b/src/ac/acl/package_yaml/check_yaml.py index dad80ac..8ac90cc 100644 --- a/src/ac/acl/package_yaml/check_yaml.py +++ b/src/ac/acl/package_yaml/check_yaml.py @@ -73,7 +73,7 @@ class CheckPackageYaml(BaseCheck): for change_file in diff_files: if change_file == package_yaml: - logger.debug("diff files: {}".format(diff_files)) + logger.debug("diff files: %s", diff_files) return True return False @@ -93,16 +93,16 @@ class CheckPackageYaml(BaseCheck): with open(os.path.join(self._work_dir, yaml_path), 'r') as yaml_data: # load yaml data self._yaml_content = yaml.safe_load(yaml_data) except IOError as e: - logging.warning("package yaml not exist. {}".format(str(e))) + logging.warning("package yaml not exist. %s", str(e)) return WARNING except yaml.YAMLError as exc: - logging.warning("Error parsering YAML: {}".format(str(exc))) + logging.warning("Error parsering YAML: %s", str(exc)) return WARNING result = SUCCESS for keyword in self.PACKAGE_YAML_NEEDED_KEY: if keyword not in self._yaml_content: - logger.error("yaml field {} missing".format(keyword)) + logger.error("yaml field %s missing", keyword) self._is_standard = True result = WARNING return result @@ -129,7 +129,7 @@ class CheckPackageYaml(BaseCheck): tags = release_tags.get_tags(sr) if not tags: - logger.warning("failed to get version by yaml, version_control: {t1}, src_repo: {t2}".format(t1=vc, t2=sr)) + logger.warning("failed to get version by yaml, version_control: %s, src_repo: %s", vc, sr) return WARNING return SUCCESS @@ -154,9 +154,9 @@ class CheckPackageYaml(BaseCheck): if not src_url: src_url = self._spec.get_source("Source") vc = self.VERSION_CTRL_TRANS.get(vc, vc) # 对特殊的版本控制对应的域名进行转换 - logger.debug("version control: {vctrl} source url: {url}".format(vctrl=vc, url=src_url)) + logger.debug("version control: %s source url: %s", vc, src_url) if vc not in src_url: # 通过判断版本控制字段是否在主页url中 判断一致性 - logger.warning("{vc} is not in url: {url}".format(vc=vc, url=src_url)) + logger.warning("%s is not in url: %s", vc, src_url) return WARNING return SUCCESS @@ -196,14 +196,14 @@ class CheckPackageYaml(BaseCheck): src_url = self._spec.get_source("Source0") if not src_url: src_url = self._spec.get_source("Source") - logger.debug("software name: {name} source url: {url}".format(name=software_name, url=src_url)) + logger.debug("software name: %s source url: %s", software_name, src_url) if software_name not in src_url: - logger.warning("{name} is not in source0: {url}".format(name=software_name, url=src_url)) + logger.warning("%s is not in source0: %s", software_name, src_url) return WARNING return SUCCESS def __call__(self, *args, **kwargs): - logger.info("check {} yaml ...".format(self._repo)) + logger.info("check %s yaml ...", self._repo) self._yaml_changed = self.is_change_package_yaml() # yaml文件变更 进行检查 # 因门禁系统限制外网访问权限，将涉及外网访问的检查功能check_repo暂时关闭 return self.start_check_with_order("fields", "repo_domain", "repo_name") diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index da04991..362c1d9 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -17,6 +17,7 @@ import os import shutil import logging +import json from src.proxy.git_proxy import GitProxy from src.ac.framework.ac_base import BaseCheck @@ -39,50 +40,22 @@ class CheckSCA(BaseCheck): """ super(CheckSCA, self).__init__(workspace, repo, conf) - self._work_diff_dir = os.path.join(workspace, "diff") # 目标目录，保存变更了的代码 - - def copy_diff_files_to_dest(self, files): - """ - 拷贝所有diff文件到目标目录 - :param files: 文件列表 - :return: - """ - for filepath in files: - try: - shutil.copy(os.path.join(self._work_dir, filepath), self._work_diff_dir) - except IOError: - logger.exception("copy {} to {} exception".format(filepath, self._work_diff_dir)) - - def save_scanoss_result(self, html): - """ - 保存结果到本地 - :param html: scanoss 结果，html格式 - :return: - """ - with open(self._scanoss_result_output, "w") as f: - f.write(html) - def check_scanoss(self): """ - scanoss工具检查代码片段引用 - https://osskb.org - https://github.com/scanoss/scanner.py - :return: + Obtain scanoss logs and result """ - gp = GitProxy(self._work_dir) - diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") - logger.debug("diff files: {}".format(diff_files)) - - self.copy_diff_files_to_dest(diff_files) - - blacklist_sbom = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../../../conf/deny_list.sbom")) - scan = ScanOSS(self._scanoss_api_key, self._scanoss_api_url, blacklist_sbom) - result = scan.scan(self._work_diff_dir) - + # Describes the reportUrl result jenkinsJobName jenkinsBuildNum prNo repoUrl of scanoss + try: + with open(self._scanoss_result_output, 'r') as f: + result_dirt = json.load(f) + except IOError: + logger.error("%s not found, make sure this file exists", self._scanoss_result_output) + return FAILED + + result = result_dirt.get('result') + # 保存详细结果到web server - if not result: - self.save_scanoss_result(scan.html) - logger.warning("click {} view scanoss detail".format(self._scanoss_result_repo_path)) + logger.warning("click %s view scanoss detail", result_dirt.get('reportUrl')) return SUCCESS if result else FAILED @@ -93,17 +66,10 @@ class CheckSCA(BaseCheck): :param kwargs: :return: """ - logger.info("check {} sca ...".format(self._repo)) + logger.info("check %s sca ...", self._repo) - logger.debug("args: {}, kwargs: {}".format(args, kwargs)) + logger.debug("args: %s, kwargs: %s", args, kwargs) scanoss_conf = kwargs.get("scanoss", {}) - self._scanoss_api_key = scanoss_conf.get("api_key", "") - self._scanoss_api_url = scanoss_conf.get("api_url", "https://osskb.org/api/scan/direct") - self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") # 保存结果到本地文件 - self._scanoss_result_repo_path = scanoss_conf.get("repo_path", "-lost linker-") # 保存结果到web server的路径 - - _ = not os.path.exists(self._work_diff_dir) and os.mkdir(self._work_diff_dir) - try: - return self.start_check() - finally: - shutil.rmtree(self._work_diff_dir) + self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") + + return self.start_check() diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index ca0b0ad..540d41f 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -53,7 +53,7 @@ class CheckSpec(BaseCheck): package_yaml = "{}.yaml".format(self._repo) # package yaml file name if len(diff_files) == 1 and diff_files[0] == package_yaml: - logger.debug("diff files: {}".format(diff_files)) + logger.debug("diff files: %s", diff_files) return True return False @@ -94,7 +94,7 @@ class CheckSpec(BaseCheck): # if lts branch, version update is forbidden if self._is_lts_branch(): - logger.debug("lts branch {}".format(self._tbranch)) + logger.debug("lts branch %s", self._tbranch) if RPMSpecAdapter.compare_version(self._spec.version, spec_o.version) == 1: logger.error("version update of lts branch is forbidden") return FAILED @@ -106,8 +106,8 @@ class CheckSpec(BaseCheck): logger.debug("revert commit") return SUCCESS - logger.error("current version: {}-r{}, last version: {}-r{}".format( - self._spec.version, self._spec.release, spec_o.version, spec_o.release)) + logger.error("current version: %s-r%s, last version: %s-r%s", + self._spec.version, self._spec.release, spec_o.version, spec_o.release) return FAILED def check_homepage(self, timeout=30, retrying=3, interval=1): @@ -119,7 +119,7 @@ class CheckSpec(BaseCheck): :return: """ homepage = self._spec.url - logger.debug("homepage: {}".format(homepage)) + logger.debug("homepage: %s", homepage) if not homepage: return SUCCESS @@ -137,15 +137,15 @@ class CheckSpec(BaseCheck): """ patches_spec = set(self._spec.patches) patches_file = set(self._gr.patch_files_not_recursive()) - logger.debug("spec patches: {}".format(patches_spec)) - logger.debug("file patches: {}".format(patches_file)) + logger.debug("spec patches: %s", patches_spec) + logger.debug("file patches: %s", patches_file) result = SUCCESS for patch in patches_spec - patches_file: - logger.error("patch {} lost".format(patch)) + logger.error("patch %s lost", patch) result = FAILED for patch in patches_file - patches_spec: - logger.warning("patch {} redundant".format(patch)) + logger.warning("patch %s redundant", patch) return result @@ -164,7 +164,7 @@ class CheckSpec(BaseCheck): content = "x86-64" if content is not None: - logger.info("exclusive arch \"{}\"".format(content)) + logger.info("exclusive arch \"%s\"", content) try: with open("exclusive_arch", "w") as f: f.write(content) @@ -189,7 +189,7 @@ class CheckSpec(BaseCheck): "last_version": spec.version, "last_release": spec.release, "compare_version": compare_version, "compare_release": compare_release, "compare": compare} - logger.info("{}".format(rs)) + logger.info("%s", rs) try: with open("pkgship_notify", "w") as f: yaml.safe_dump(rs, f) @@ -197,7 +197,7 @@ class CheckSpec(BaseCheck): logger.exception("save pkgship exception") def __call__(self, *args, **kwargs): - logger.info("check {} spec ...".format(self._repo)) + logger.info("check %s spec ...", self._repo) self._ex_exclusive_arch() self._tbranch = kwargs.get("tbranch", None) diff --git a/src/ac/common/gitee_repo.py b/src/ac/common/gitee_repo.py index 996bfe4..c24ca09 100644 --- a/src/ac/common/gitee_repo.py +++ b/src/ac/common/gitee_repo.py @@ -53,16 +53,16 @@ class GiteeRepo(object): for filename in filenames: rel_file_path = os.path.join(dirpath, filename).replace(self._work_dir, "").lstrip("/") if self.is_compress_file(filename): - logger.debug("find compress file: {}".format(rel_file_path)) + logger.debug("find compress file: %s", rel_file_path) self._compress_files.append(rel_file_path) elif self.is_patch_file(filename): - logger.debug("find patch file: {}".format(rel_file_path)) + logger.debug("find patch file: %s", rel_file_path) self._patch_files.append(rel_file_path) elif self.is_spec_file(filename): - logger.debug("find spec file: {}".format(rel_file_path)) + logger.debug("find spec file: %s", rel_file_path) spec_files.append(filename) elif self.is_package_yaml_file(filename): - logger.debug("find yaml file: {}".format(rel_file_path)) + logger.debug("find yaml file: %s", rel_file_path) self.yaml_file = rel_file_path def guess_real_spec_file(): @@ -109,7 +109,7 @@ class GiteeRepo(object): decompress_cmd = "cd {}; timeout 120s tar -C {} -xavf {}".format( self._work_dir, self._decompress_dir, file_path) else: - logger.warning("unsupport compress file: {}".format(file_path)) + logger.warning("unsupport compress file: %s", file_path) return False ret, _, _ = shell_cmd_live(decompress_cmd) @@ -136,20 +136,20 @@ class GiteeRepo(object): :param patch: 补丁 :param max_leading: leading path """ - logger.debug("apply patch {}".format(patch)) + logger.debug("apply patch %s", patch) for patch_dir in [filename for filename in os.listdir(self._decompress_dir) if os.path.isdir(os.path.join(self._decompress_dir, filename))] + ["."]: if patch_dir.startswith(".git"): continue for leading in range(max_leading + 1): - logger.debug("try dir {} -p{}".format(patch_dir, leading)) + logger.debug("try dir %s -p%s", patch_dir, leading) if GitProxy.apply_patch_at_dir(os.path.join(self._decompress_dir, patch_dir), os.path.join(self._work_dir, patch), leading): logger.debug("patch success") self.patch_dir_mapping[patch] = os.path.join(self._decompress_dir, patch_dir) return True - logger.info("apply patch {} failed".format(patch)) + logger.info("apply patch %s failed", patch) return False def apply_all_patches(self, *patches): @@ -166,7 +166,7 @@ class GiteeRepo(object): if patch in set(self._patch_files): rs.append(self.apply_patch(patch)) else: - logger.error("patch {} not exist".format(patch)) + logger.error("patch %s not exist", patch) rs.append(False) return 0 if all(rs) else (1 if any(rs) else -1) diff --git a/src/ac/common/linter.py b/src/ac/common/linter.py index 0b76303..95eacc8 100644 --- a/src/ac/common/linter.py +++ b/src/ac/common/linter.py @@ -73,13 +73,13 @@ class LinterCheck(object): * (E) error, for probable bugs in the code * (F) fatal, if an error occurred which prevented pylint from doing """ - logger.debug("check python file: {}".format(filepath)) + logger.debug("check python file: %s", filepath) # E0401: import module error pylint_cmd = "pylint3 --disable=E0401 {}".format(filepath) ret, out, _ = shell_cmd_live(pylint_cmd, cap_out=True, verbose=True) if ret: - logger.debug("pylint ret, {}".format(ret)) + logger.debug("pylint ret, %s", ret) return cls.get_summary_of_pylint(out) @@ -88,12 +88,12 @@ class LinterCheck(object): """ Check golang code by golint """ - logger.debug("check go file: {}".format(filepath)) + logger.debug("check go file: %s", filepath) golint_cmd = "golint {}".format(filepath) ret, out, _ = shell_cmd_live(golint_cmd, cap_out=True, verbose=True) if ret: - logger.debug("golint error, {}".format(ret)) + logger.debug("golint error, %s", ret) return {} return cls.get_summary_of_golint(out) @@ -103,13 +103,13 @@ class LinterCheck(object): """ Check c/c++ code by splint """ - logger.debug("check c/c++ file: {}".format(filepath)) + logger.debug("check c/c++ file: %s", filepath) splint_cmd = "splint {}".format(filepath) #ret, out, _ = shell_cmd_live(splint_cmd, cap_out=True, verbose=True) ret, out, _ = shell_cmd(splint_cmd) if ret: - logger.debug("splint error, {}".format(ret)) + logger.debug("splint error, %s", ret) return {} return cls.get_summary_of_splint(out) diff --git a/src/ac/common/rpm_spec_adapter.py b/src/ac/common/rpm_spec_adapter.py index a714476..f32ddba 100644 --- a/src/ac/common/rpm_spec_adapter.py +++ b/src/ac/common/rpm_spec_adapter.py @@ -70,7 +70,7 @@ class RPMSpecAdapter(object): """ try: value = self.buildarch - logger.debug("build arch: {}".format(value)) + logger.debug("build arch: %s", value) if "x86_64" in value.lower(): return True @@ -85,7 +85,7 @@ class RPMSpecAdapter(object): """ try: value = self.buildarch - logger.debug("build arch: {}".format(value)) + logger.debug("build arch: %s", value) if "aarch64" in value.lower(): return True @@ -111,7 +111,7 @@ class RPMSpecAdapter(object): version_n = "{}{}".format(version_n, '.0' * (len(version_o.split('.')) - len(version_n.split('.')))) version_o = "{}{}".format(version_o, '.0' * (len(version_n.split('.')) - len(version_o.split('.')))) - logger.debug("compare versions: {} vs {}".format(version_n, version_o)) + logger.debug("compare versions: %s vs %s", version_n, version_o) z = zip(version_n.split("."), version_o.split(".")) for p in z: @@ -121,7 +121,7 @@ class RPMSpecAdapter(object): elif int(p[0]) > int(p[1]): return 1 except ValueError as exc: - logger.debug("check version exception, {}".format(exc)) + logger.debug("check version exception, %s", exc) continue return 0 diff --git a/src/ac/common/scanoss.py b/src/ac/common/scanoss.py index 39ce039..2262702 100644 --- a/src/ac/common/scanoss.py +++ b/src/ac/common/scanoss.py @@ -50,7 +50,7 @@ class ScanOSS(object): try: json_format = json.loads(result) except ValueError: - logger.exception("illegal scanoss result, \"{}\"".format(result)) + logger.exception("illegal scanoss result, \"%s\"", result) return True snippets = 0 @@ -67,7 +67,7 @@ class ScanOSS(object): detail_trs.append(self.__class__.detail_trs(filename, item)) - logger.debug("snippets: {}, files: {}".format(snippets, files)) + logger.debug("snippets: %s, files: %s", snippets, files) detail = "{th}{trs}
".format( th=self.__class__.detail_th(), trs="\n".join(detail_trs)) @@ -161,7 +161,7 @@ class ScanOSS(object): :param directory: 需要扫描的目录 :return: """ - logger.debug("scan dir: {}".format(directory)) + logger.debug("scan dir: %s", directory) #scanoss_cmd = "scanner.py --format {} {} --apiurl {} {}".format( # "plain", "--key {}".format(self._key) if self._key else "", self._api_url, directory) scanoss_cmd = "scanner.py --blacklist {} --format {} {} --apiurl {} {}".format( @@ -169,8 +169,8 @@ class ScanOSS(object): ret, out, err = shell_cmd(scanoss_cmd) if ret: - logger.error("scanoss error, {}".format(ret)) - logger.error("{}".format(err)) + logger.error("scanoss error, %s", ret) + logger.error("%s", err) return True return self.result_analysis(out) diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index 6e32ebb..f208b00 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -49,7 +49,7 @@ class AC(object): self.load_check_elements_from_acl_directory(acl_path) self.load_check_elements_from_conf(conf, community) - logger.debug("check list: {}".format(self._ac_check_elements)) + logger.debug("check list: %s", self._ac_check_elements) @staticmethod def is_repo_support_check(repo, check_element): @@ -74,7 +74,7 @@ class AC(object): """ for element in self._ac_check_elements: check_element = self._ac_check_elements[element] - logger.debug("check {}".format(element)) + logger.debug("check %s", element) # show in gitee, must starts with "check_" hint = check_element.get("hint", "check_{}".format(element)) @@ -82,25 +82,25 @@ class AC(object): hint = "check_{}".format(hint) if not self.__class__.is_repo_support_check(repo, check_element): - logger.debug("{} not support check".format(repo)) + logger.debug("%s not support check", repo) continue # import module module_path = check_element.get("module", "{}.check_{}".format(element, element)) # eg: spec.check_spec try: module = importlib.import_module("." + module_path, self._acl_package) - logger.debug("load module {} succeed".format(module_path)) + logger.debug("load module %s succeed", module_path) except ImportError as exc: - logger.exception("import module {} exception, {}".format(module_path, exc)) + logger.exception("import module %s exception, %s", module_path, exc) continue # import entry entry_name = check_element.get("entry", "Check{}".format(element.capitalize())) try: entry = getattr(module, entry_name) - logger.debug("load entry \"{}\" succeed".format(entry_name)) + logger.debug("load entry \"%s\" succeed", entry_name) except AttributeError as exc: - logger.warning("entry \"{}\" not exist in module {}, {}".format(entry_name, module_path, exc)) + logger.warning("entry \"%s\" not exist in module %s, %s", entry_name, module_path, exc) continue # new a instance @@ -109,26 +109,26 @@ class AC(object): entry = entry(workspace, repo, check_element) # new a instance except Exception as exc: self._ac_check_result.append({"name": hint, "result": FAILED.val}) - logger.exception("new a instance of class {} exception, {}".format(entry_name, exc)) + logger.exception("new a instance of class %s exception, %s", entry_name, exc) continue if not callable(entry): # check callable - logger.warning("entry {} not callable".format(entry_name)) + logger.warning("entry %s not callable", entry_name) continue # do ac check try: result = entry(**kwargs) - logger.debug("check result {} {}".format(element, result)) + logger.debug("check result %s %s", element, result) except Exception as exc: - logger.exception("check exception, {} {}".format(element, exc)) + logger.exception("check exception, %s %s", element, exc) continue self._ac_check_result.append({"name": hint, "result": result.val}) dataset.set_attr("access_control.build.acl.{}".format(element), result.hint) dataset.set_attr("access_control.build.content", self._ac_check_result) - logger.debug("ac result: {}".format(self._ac_check_result)) + logger.debug("ac result: %s", self._ac_check_result) def load_check_elements_from_acl_directory(self, acl_dir): """ @@ -150,18 +150,18 @@ class AC(object): with open(conf_file, "r") as f: content = yaml.safe_load(f) except IOError: - logger.exception("ac conf file {} not exist".format(conf_file)) + logger.exception("ac conf file %s not exist", conf_file) return except YAMLError: logger.exception("illegal conf file format") return elements = content.get(community, {}) - logger.debug("community \"{}\" conf: {}".format(community, elements)) + logger.debug("community \"%s\" conf: %s", community, elements) for name in elements: if name in self._ac_check_elements: if elements[name].get("exclude"): - logger.debug("exclude: {}".format(name)) + logger.debug("exclude: %s", name) self._ac_check_elements.pop(name) else: self._ac_check_elements[name] = elements[name] @@ -172,7 +172,7 @@ class AC(object): :param ac_file: :return: """ - logger.debug("save ac result to file {}".format(ac_file)) + logger.debug("save ac result to file %s", ac_file) with open(ac_file, "w") as f: f.write("ACL={}".format(json.dumps(self._ac_check_result))) @@ -202,15 +202,12 @@ def init_args(): parser.add_argument("-l", type=str, dest="trigger_link", help="job trigger link") # scanoss - parser.add_argument("--scanoss-api-key", type=str, dest="scanoss_api_key", help="scanoss api key") - parser.add_argument("--scanoss-api-url", type=str, dest="scanoss_api_url", - default="https://osskb.org/api/scan/direct", help="scanoss api url") parser.add_argument("--scanoss-output", type=str, dest="scanoss_output", default="scanoss_result", help="scanoss result output") - parser.add_argument("--scanoss-repo-path", type=str, dest="scanoss_repo", help="scanoss result repo path") + parser.add_argument("--codecheck-api-key", type=str, dest="codecheck_api_key", help="codecheck api key") parser.add_argument("--codecheck-api-url", type=str, dest="codecheck_api_url", - default="http://124.71.75.234:8384/api/openlibing/codecheck/start", help="codecheck api url") + default="https://majun.osinfra.cn:8384/api/openlibing/codecheck", help="codecheck api url") return parser.parse_args() @@ -225,10 +222,10 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("ac") - logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.community, args.repo)) + logger.info("using credential %s", args.account.split(":")[0]) + logger.info("cloning repository https://gitee.com/%s/%s.git ", args.community, args.repo) logger.info("clone depth 4") - logger.info("checking out pull request {}".format(args.pr)) + logger.info("checking out pull request %s", args.pr) # notify gitee from src.proxy.gitee_proxy import GiteeProxy @@ -289,11 +286,10 @@ if "__main__" == __name__: gp.create_tags_of_pr(args.pr, "ci_processing") # scanoss conf - scanoss = {"api_key": args.scanoss_api_key, "api_url": args.scanoss_api_url, - "output": args.scanoss_output, "repo_path": args.scanoss_repo} + scanoss = {"output": args.scanoss_output} codecheck = {"pr_url": "https://gitee.com/{}/{}/pulls/{}".format(args.community, args.repo, args.pr), - "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url + "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url, "codecheck_api_key": args.codecheck_api_key } # build diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 85fce10..5cabb6f 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -42,28 +42,3 @@ openeuler: module: openlibing.check_code entry: CheckCode allow_list: ["pkgship", "kunpengsecl", "release-tools"] -zhengyaohui: - spec: - hint: check_spec_file - module: spec.check_spec - entry: CheckSpec - ignored: ["homepage"] - code: - hint: check_code_style - module: code.check_code_style - entry: CheckCodeStyle - #exclude: True - ignored: ["patch"] - package_yaml: - hint: check_package_yaml_file - module: package_yaml.check_yaml - entry: CheckPackageYaml - ignored: ["fields"] - package_license: - hint: check_package_license - module: package_license.check_license - entry: CheckLicense - sca: - exclude: True - openlibing: - exclude: True diff --git a/src/ac/framework/ac_base.py b/src/ac/framework/ac_base.py index 7deb3b5..9f3e9d7 100644 --- a/src/ac/framework/ac_base.py +++ b/src/ac/framework/ac_base.py @@ -57,27 +57,27 @@ class BaseCheck(object): result = SUCCESS for name in items: try: - logger.debug("check {}".format(name)) + logger.debug("check %s", name) method = getattr(self, "check_{}".format(name)) rs = method() - logger.debug("{} -> {}".format(name, rs)) + logger.debug("%s -> %s", name, rs) except Exception as e: # 忽略代码错误 - logger.exception("internal error: {}".format(e)) + logger.exception("internal error: %s", e) continue ignored = True if self._conf and name in self._conf.get("ignored", []) else False - logger.debug("{} ignore: {}".format(name, ignored)) + logger.debug("%s ignore: %s", name, ignored) if rs is SUCCESS: - logger.info("check {:<30}pass".format(name)) + logger.info("check %s pass", name) elif rs is WARNING: - logger.warning("check {:<30}warning{}".format(name, " [ignored]" if ignored else "")) + logger.warning("check %s warning %s", name, " [ignored]" if ignored else "") elif rs is FAILED: - logger.error("check {:<30}fail{}".format(name, " [ignored]" if ignored else "")) + logger.error("check %s fail %s", name, " [ignored]" if ignored else "") else: # never here - logger.exception("check {:<30}exception{}".format(name, " [ignored]" if ignored else "")) + logger.exception("check %s exception %s", name, " [ignored]" if ignored else "") continue if not ignored: @@ -91,6 +91,6 @@ class BaseCheck(object): """ members = inspect.getmembers(self, inspect.ismethod) items = [member[0].replace("check_", "") for member in members if member[0].startswith("check_")] - logger.debug("check items: {}".format(items)) + logger.debug("check items: %s", items) return self.start_check_with_order(*items) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index fabafd0..149b701 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -49,7 +49,7 @@ class ExtraWork(object): try: with open(pkgship_meta_path, "r") as f: pkgship_meta = yaml.safe_load(f) - logger.debug("pkgship meta: {}".format(pkgship_meta)) + logger.debug("pkgship meta: %s", pkgship_meta) if pkgship_meta.get("compare_version") == 1: # version upgrade logger.debug("pkgship: notify") return True @@ -91,11 +91,10 @@ class ExtraWork(object): :return: """ #get rpms - logging.info("compare package start") curr_rpm = self._rpm_package.main_package_local() last_rpm = self._rpm_package.last_main_package(package_arch, package_url) - logger.debug("curr_rpm: {}".format(curr_rpm)) - logger.debug("last_rpm: {}".format(last_rpm)) + logger.debug("curr_rpm: %s", curr_rpm) + logger.debug("last_rpm: %s", last_rpm) if not curr_rpm or not last_rpm: logger.info("no rpms") return @@ -109,8 +108,8 @@ class ExtraWork(object): debuginfos = None curr_rpm_debug = self._rpm_package.debuginfo_package_local() last_rpm_debug = self._rpm_package.last_debuginfo_package(package_arch, package_url) - logger.debug("curr_rpm_debug: {}".format(curr_rpm_debug)) - logger.debug("last_rpm_debug: {}".format(last_rpm_debug)) + logger.debug("curr_rpm_debug: %s", curr_rpm_debug) + logger.debug("last_rpm_debug: %s", last_rpm_debug) if curr_rpm_debug and last_rpm_debug: debuginfos = [last_rpm_debug, curr_rpm_debug] @@ -124,9 +123,9 @@ class ExtraWork(object): check_abi = CheckAbi(result_output_file=output, input_rpms_path=related_rpms_url) ret = check_abi.process_with_rpm(rpms, debuginfos) if ret == 1: - logger.error("check abi error: {}".format(ret)) + logger.error("check abi error: %s", ret) else: - logger.debug("check abi ok: {}".format(ret)) + logger.debug("check abi ok: %s", ret) if os.path.exists(output): # change of abi @@ -136,7 +135,7 @@ class ExtraWork(object): else: comment = {"name": "check_abi/{}/{}".format(package_arch, self._repo), "result": "SUCCESS"} - logger.debug("check abi comment: {}".format(comment)) + logger.debug("check abi comment: %s", comment) try: with open(comment_file, "r") as f: # one repo with multi build package comments = yaml.safe_load(f) @@ -152,77 +151,13 @@ class ExtraWork(object): logger.exception("yaml load check abi comment file exception") comments.append(comment) - logger.debug("check abi comments: {}".format(comments)) - logging.info("compare package finish") + logger.debug("check abi comments: %s", comments) try: with open(comment_file, "w") as f: yaml.safe_dump(comments, f) # list except IOError: logger.exception("save check abi comment exception") - def compare_rpm_diff(self, package_url, package_arch, output, committer, comment_file, obs_addr, - branch_name="master", obs_repo_url=None): - """ - 对比两个版本rpm包之间的差异，根据差异找到受影响的rpm包 - :param package_arch: - :param obs_repo_url: - :return: - """ - #get rpms - logging.info("compare package start") - - logging.info("compare subpackage start:gcc-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-10.3.1-20210914.5.oe1.x86_64.rpm") - logging.info("compare abi pass") - logging.info("compare config pass") - logging.info("compare filelist pass") - logging.info("compare requires pass") - logging.info("compare provide pass") - logging.info("compare subpackage finish:gcc-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-10.3.1-20210914.5.oe1.x86_64.rpm") - - logging.info("compare subpackage start:gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm") - logging.info("compare abi pass") - logging.info("compare config pass") - logging.info("compare filelist pass") - logging.info("compare requires pass") - logging.info("compare provide pass") - logging.info("compare subpackage finish:gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-c++-10.3.1-20210914.5.oe1.x86_64.rpm") - - logging.info("compare subpackage start:gcc-debuginfo-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-deebuginfo-10.3.1-20210914.5.oe1.x86_64.rpm") - logging.info("compare abi pass") - logging.info("compare config pass") - logging.info("compare filelist pass") - logging.info("compare requires pass") - logging.info("compare provide pass") - logging.info("compare subpackage finish:gcc-debuginfo-10.3.1-20210914.5.oe1.x86_64.rpm and gcc-debuginfo-10.3.1-20210914.5.oe1.x86_64.rpm") - - comment = {"name": "compare_package/{}/{}".format(package_arch, self._repo), "result": "SUCCESS", - "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, - os.path.basename(output), package_url)} - - logger.debug("compare package comment: {}".format(comment)) - try: - with open(comment_file, "r") as f: # one repo with multi build package - comments = yaml.safe_load(f) - except IOError as e: - logger.debug("no history compare package comment") - - comments = [] - if os.path.exists(comment_file): - try: - with open(comment_file, "r") as f: # one repo with multi build package - comments = yaml.safe_load(f) - except IOError: - logger.exception("yaml load compare package comment file exception") - - comments.append(comment) - logger.debug("compare package comments: {}".format(comments)) - logging.info("compare package finish") - try: - with open(comment_file, "w") as f: - yaml.safe_dump(comments, f) # list - except IOError: - logger.exception("save compare package comment exception") - def check_install_rpm(self, branch_name, arch, install_root): """ 检查生成的rpm是否可以安装 @@ -235,14 +170,14 @@ class ExtraWork(object): # 1. prepare install root directory _ = not os.path.exists(install_root) and os.mkdir(install_root) - logger.info("create install root directory: {}".format(install_root)) + logger.info("create install root directory: %s", install_root) repo_name_prefix = "check_install" # 2. prepare repo repo_source = OBSRepoSource("http://119.3.219.20:82") # obs 实时构建repo地址 repo_config = repo_source.generate_repo_info(branch_name, arch, "check_install") - logger.info("repo source config:\n{}".format(repo_config)) + logger.info("repo source config:\n%s", repo_config) # write to /etc/yum.repos.d with open("obs_realtime.repo", "w+") as f: @@ -254,18 +189,18 @@ class ExtraWork(object): for name, package in self._rpm_package.iter_all_rpm(): # ignore debuginfo rpm if "debuginfo" in name or "debugsource" in name: - logger.debug("ignore debug rpm: {}".format(name)) + logger.debug("ignore debug rpm: %s", name) continue names.append(name) packages.append(package) - logger.info("install rpms: {}".format(names)) + logger.info("install rpms: %s", names) if packages: check_install_cmd = "sudo dnf install -y --installroot={} --setopt=reposdir=. {}".format( install_root, " ".join(packages)) ret, _, err = shell_cmd_live(check_install_cmd, verbose=True) if ret: - logger.error("install rpms error, {}, {}".format(ret, err)) + logger.error("install rpms error, %s, %s", ret, err) else: logger.info("install rpm success") @@ -317,9 +252,7 @@ if "__main__" == __name__: args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) elif args.func == "checkabi": # run before copy rpm to rpm repo - #ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, - #args.obs_addr, args.branch_name, args.obs_repo_url) - ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, + ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, args.obs_addr, args.branch_name, args.obs_repo_url) elif args.func == "checkinstall": ew.check_install_rpm(args.branch_name, args.arch, args.install_root) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 3622b4f..5d778bc 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -82,7 +82,7 @@ class Comment(object): base_job_name = os.environ.get("JOB_NAME") base_build_id = os.environ.get("BUILD_ID") base_build_id = int(base_build_id) - logger.debug("base_job_name: {}, base_build_id: {}".format(base_job_name, base_build_id)) + logger.debug("base_job_name: %s, base_build_id: %s", base_job_name, base_build_id) base_build = jenkins_proxy.get_build(base_job_name, base_build_id) logger.debug("get base build") self._up_builds = jenkins_proxy.get_upstream_builds(base_build) @@ -103,8 +103,7 @@ class Comment(object): comments.extend(self._comment_of_ac(self._up_up_builds[0])) if self._up_builds: comments.extend(self._comment_of_build(self._up_builds)) - #comments.extend(self._comment_of_check_abi(self._up_builds)) - comments.extend(self._comment_of_compare_package(self._up_builds)) + comments.extend(self._comment_of_check_abi(self._up_builds)) comments.append("") return comments @@ -121,7 +120,7 @@ class Comment(object): try: acl = json.loads(os.environ["ACL"]) - logger.debug("ac result: {}".format(acl)) + logger.debug("ac result: %s", acl) except ValueError: logger.exception("invalid ac result format") return [] @@ -139,7 +138,7 @@ class Comment(object): comments.append(self.__class__.comment_html_table_tr_rowspan( item["name"], ac_result.emoji, ac_result.hint)) - logger.info("ac comment: {}".format(comments)) + logger.info("ac comment: %s", comments) return comments @@ -159,52 +158,7 @@ class Comment(object): comments.append(self.__class__.comment_html_table_tr( name, ac_result.emoji, ac_result.hint, "{}{}".format(build_url, "console"), build.buildno)) - logger.info("build comment: {}".format(comments)) - - return comments - - def _comment_of_compare_package(self, builds): - """ - compare package comment - :param builds: - :return: - """ - comments = [] - - def match(name, comment_file): - if "aarch64" in name and "aarch64" in comment_file: - return True - if "x86-64" in name and "x86_64" in comment_file: - return True - return False - - for check_abi_comment_file in self._check_abi_comment_files: - logger.debug("compare package comment file: {}".format(check_abi_comment_file)) - if not os.path.exists(check_abi_comment_file): # check abi评论文件存在 - continue - for build in builds: - name = build.job._data["fullName"] - logger.debug("check build {}".format(name)) - if not match(name, check_abi_comment_file): # 找到匹配的jenkins build - continue - logger.debug("build \"{}\" match".format(name)) - - status = build.get_status() - logger.debug("build state: {}".format(status)) - if ACResult.get_instance(status) == SUCCESS: # 保证build状态成功 - with open(check_abi_comment_file, "r") as f: - try: - content = yaml.safe_load(f) - except YAMLError: # yaml base exception - logger.exception("illegal yaml format of check abi comment file ") - logger.debug("comment: {}".format(content)) - for item in content: - ac_result = ACResult.get_instance(item.get("result")) - comments.append(self.__class__.comment_html_table_tr( - "compare_package", ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) - break - - logger.info("compare package comment: {}".format(comments)) + logger.info("build comment: %s", comments) return comments @@ -224,25 +178,25 @@ class Comment(object): return False for check_abi_comment_file in self._check_abi_comment_files: - logger.debug("check abi comment file: {}".format(check_abi_comment_file)) + logger.debug("check abi comment file: %s", check_abi_comment_file) if not os.path.exists(check_abi_comment_file): # check abi评论文件存在 continue for build in builds: name = build.job._data["fullName"] - logger.debug("check build {}".format(name)) + logger.debug("check build %s", name) if not match(name, check_abi_comment_file): # 找到匹配的jenkins build continue - logger.debug("build \"{}\" match".format(name)) + logger.debug("build \"%s\" match", name) status = build.get_status() - logger.debug("build state: {}".format(status)) + logger.debug("build state: %s", status) if ACResult.get_instance(status) == SUCCESS: # 保证build状态成功 with open(check_abi_comment_file, "r") as f: try: content = yaml.safe_load(f) except YAMLError: # yaml base exception logger.exception("illegal yaml format of check abi comment file ") - logger.debug("comment: {}".format(content)) + logger.debug("comment: %s", content) for item in content: ac_result = ACResult.get_instance(item.get("result")) comments.append(self.__class__.comment_html_table_tr( @@ -250,7 +204,7 @@ class Comment(object): "markdown" if "link" in item else "", hashtag=False)) break - logger.info("check abi comment: {}".format(comments)) + logger.info("check abi comment: %s", comments) return comments diff --git a/src/build/obs_repo_source.py b/src/build/obs_repo_source.py index 3548350..638e44c 100644 --- a/src/build/obs_repo_source.py +++ b/src/build/obs_repo_source.py @@ -69,35 +69,35 @@ class OBSRepoSource(object): # openstack common url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part_common, arch) if do_requests("GET", url) == 0: - logger.debug("add openstack common repo: {}".format(url)) + logger.debug("add openstack common repo: %s", url) repo_config += self.repo_format("openstack_common", repo_name_prefix + "_openstack_common", url) # openstack base url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part_base, arch) if do_requests("GET", url) == 0: - logger.debug("add openstack base repo: {}".format(url)) + logger.debug("add openstack base repo: %s", url) repo_config += self.repo_format("openstack_base", repo_name_prefix + "_openstack_base", url) else: obs_path_part = branch.replace("-", ":/") - logger.debug("branch={}, obs_path_part={}".format(branch, obs_path_part)) + logger.debug("branch=%s, obs_path_part=%s", branch, obs_path_part) # main url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part, arch) if do_requests("GET", url) == 0: - logger.debug("add main repo: {}".format(url)) + logger.debug("add main repo: %s", url) repo_config += self.repo_format(repo_name_prefix + "_main", repo_name_prefix + "_main", url) # epol url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part + ":/Epol", arch) if do_requests("GET", url) == 0: - logger.debug("add epol repo: {}".format(url)) + logger.debug("add epol repo: %s", url) repo_config += self.repo_format(repo_name_prefix + "_epol", repo_name_prefix + "_epol", url) # extras url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part + ":/Extras", arch) if do_requests("GET", url) == 0: - logger.debug("add extras repo: {}".format(url)) + logger.debug("add extras repo: %s", url) repo_config += self.repo_format(repo_name_prefix + "_extras", repo_name_prefix + "_extras", url)\ return repo_config diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 1e3e354..be38c69 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -46,7 +46,8 @@ class SinglePackageBuild(object): "oepkg_openstack-rocky_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:rocky"], "oepkg_openstack-common_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:common"], "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"], - "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"] + "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"], + "openEuler-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3", "openEuler:20.03:LTS:SP3:Epol"] } BUILD_IGNORED_GITEE_BRANCH = ["riscv"] @@ -70,11 +71,12 @@ class SinglePackageBuild(object): """ return OBSProxy.list_repos_of_arch(project, self._package, self._arch, show_exclude=True) - def build_obs_repos(self, project, repos, work_dir, code_dir): + def build_obs_repos(self, project, repos, spec, work_dir, code_dir): """ build :param project: 项目名 :param repos: obs repo + :param spec: 指定spec文件 :param code_dir: 码云代码在本地路径 :param work_dir: :return: @@ -100,16 +102,16 @@ class SinglePackageBuild(object): # osc build for repo in repos: if repo["state"] == "excluded" and repo["mpac"] == "raspberrypi-kernel": - logger.info("repo {}:{} excluded".format(repo["repo"], repo["mpac"])) + logger.info("repo %s:%s excluded", repo["repo"], repo["mpac"]) continue root_build = repo["mpac"] in self.PACKAGES_USE_ROOT if not OBSProxy.build_package( - project, self._package, repo["repo"], self._arch, repo["mpac"], + project, self._package, repo["repo"], self._arch, spec, repo["mpac"], root_build=root_build, disable_cpio=True): - logger.error("build {} ... failed".format(repo["repo"])) + logger.error("build %s ... failed", repo["repo"]) return 3 - logger.info("build {} ... ok".format(repo["repo"])) + logger.info("build %s ... ok", repo["repo"]) logger.debug("build all repos ... finished") @@ -187,44 +189,45 @@ class SinglePackageBuild(object): ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: - logger.error("prepare build environ error, {}".format(ret)) + logger.error("prepare build environ error, %s", ret) return False return True - def build(self, work_dir, code_dir): + def build(self, spec, work_dir, code_dir): """ 入口 + :param spec: 指定spec文件 :param work_dir: obs工作目录 :param code_dir: 代码目录 :return: """ if self._branch in self.BUILD_IGNORED_GITEE_BRANCH: - logger.error("branch \"{}\" ignored".format(self._branch)) + logger.error("branch \"%s\" ignored", self._branch) return 0 if self._branch not in self.GITEE_BRANCH_PROJECT_MAPPING: - logger.error("branch \"{}\" not support yet".format(self._branch)) + logger.error("branch \"%s\" not support yet", self._branch) return 1 has_any_repo_build = False for project in self.GITEE_BRANCH_PROJECT_MAPPING.get(self._branch): - logger.debug("start build project {}".format(project)) + logger.debug("start build project %s", project) obs_repos = self.get_need_build_obs_repos(project) if not obs_repos: - logger.info("all repos ignored of project {}".format(project)) + logger.info("all repos ignored of project %s", project) continue - logger.debug("build obs repos: {}".format(obs_repos)) + logger.debug("build obs repos: %s", obs_repos) has_any_repo_build = True - ret = self.build_obs_repos(project, obs_repos, work_dir, code_dir) + ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) if ret > 0: - logger.debug("build run return {}".format(ret)) - logger.error("build {} {} {} ... {}".format(project, self._package, self._arch, "failed")) + logger.debug("build run return %s", ret) + logger.error("build %s %s %s ... %s", project, self._package, self._arch, "failed") return 1 # finish if any error else: - logger.info("build {} {} {} ... {}".format(project, self._package, self._arch, "ok")) + logger.info("build %s %s %s ... %s", project, self._package, self._arch, "ok") # if no repo build, regard as fail if not has_any_repo_build: @@ -253,6 +256,7 @@ def init_args(): parser.add_argument("-t", type=str, dest="account", help="gitee account") parser.add_argument("-o", type=str, dest="owner", default="src-openeuler", help="gitee owner") + parser.add_argument("--spec", type=str, dest="spec", default="", help="spec files") return parser.parse_args() @@ -265,10 +269,10 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") - logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.owner, args.repo)) + logger.info("using credential %s", args.account.split(":")[0]) + logger.info("cloning repository https://gitee.com/%s/%s.git ", args.owner, args.repo) logger.info("clone depth 1") - logger.info("checking out pull request {}".format(args.pr)) + logger.info("checking out pull request %s", args.pr) from src.utils.dist_dataset import DistDataset from src.proxy.git_proxy import GitProxy @@ -315,7 +319,7 @@ if "__main__" == __name__: dd.set_attr_stime("spb.build.stime") spb = SinglePackageBuild(args.package, args.arch, args.branch) - rs = spb.build(args.workspace, args.code) + rs = spb.build(args.spec, args.workspace, args.code) dd.set_attr("spb.build.result", "failed" if rs else "successful") dd.set_attr_etime("spb.build.etime") diff --git a/src/jobs/jenkins_job.py b/src/jobs/jenkins_job.py index 3978493..811058c 100755 --- a/src/jobs/jenkins_job.py +++ b/src/jobs/jenkins_job.py @@ -59,7 +59,7 @@ class JenkinsJobs(object): :param interval: 每次batch请求后sleep时间（秒）， :return: """ - logger.info("{} jobs {}".format(action, jobs)) + logger.info("%s jobs %s", action, jobs) real_jobs = self.get_real_target_jobs(jobs, exclude_jobs if exclude_jobs else []) def run_once(target_jobs): @@ -71,14 +71,14 @@ class JenkinsJobs(object): for index in range(batch): works = [gevent.spawn(self.dispatch, action, job, jenkins_proxy) for job in target_jobs[index * concurrency: (index + 1) * concurrency]] - logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) + logger.info("%s works, %s/%s ", len(works), index + 1, batch) gevent.joinall(works) for work in works: if work.value["result"]: - logger.info("{} job {} ... ok".format(action, work.value["job"])) + logger.info("%s job %s ... ok", action, work.value["job"]) else: _failed_jobs.append(work.value["job"]) - logger.error("{} job {} ... failed".format(action, work.value["job"])) + logger.error("%s job %s ... failed", action, work.value["job"]) time.sleep(interval) @@ -89,12 +89,12 @@ class JenkinsJobs(object): for index in range(retry): if not failed_jobs: break - logger.info("{} jobs failed, retrying {}/{}".format(len(failed_jobs), index + 1, retry)) + logger.info("%s jobs failed, retrying %s/%s", len(failed_jobs), index + 1, retry) failed_jobs = run_once(failed_jobs) if failed_jobs: - logger.warning("{} failed jobs".format(len(failed_jobs))) - logger.warning("{}{}".format(",".join(failed_jobs[:100]), "..." if len(failed_jobs) > 100 else "")) + logger.warning("%s failed jobs", len(failed_jobs)) + logger.warning("%s%s", ",".join(failed_jobs[:100]), "..." if len(failed_jobs) > 100 else "") def dispatch(self, action, job, jenkins_proxy): """ @@ -148,7 +148,7 @@ class SrcOpenEulerJenkinsJobs(JenkinsJobs): with open(os.path.join(exclusive_arch_path, filename), "r") as f: arches = f.readline() self._exclusive_arch[filename] = [arch.strip() for arch in arches.split(",")] - logger.debug("exclusive arch: {}".format(self._exclusive_arch)) + logger.debug("exclusive arch: %s", self._exclusive_arch) def get_real_target_jobs(self, jobs, exclude_jobs): """ @@ -274,7 +274,7 @@ class OpenEulerJenkinsJobs(SrcOpenEulerJenkinsJobs): # build script = self.guess_build_script(buddy["repo"]) - logger.debug("guess build script: {}".format("script")) + logger.debug("guess build script: %s", "script") ele = root.findall("buiders/hudson.task.Shell/command") if ele: # replace first command diff --git a/src/jobs/obs_meta_strategy.py b/src/jobs/obs_meta_strategy.py index 6b30620..24b0c40 100755 --- a/src/jobs/obs_meta_strategy.py +++ b/src/jobs/obs_meta_strategy.py @@ -79,20 +79,20 @@ class ObsMetaStrategy(object): if filename == "_service": _service = os.path.join(dirpath, filename) try: - logger.debug("analysis {}".format(_service)) + logger.debug("analysis %s", _service) tree = ET.parse(_service) elements = tree.findall(".//param[@name=\"url\"]") # next/openEuler/zip except: - logger.exception("invalid xml format, {}".format(_service)) + logger.exception("invalid xml format, %s", _service) continue _repos = [element.text.strip("/").split("/")[-1] for element in elements] # eg: next/openEuler/zip - logger.debug("get repos: {}".format(_repos)) + logger.debug("get repos: %s", _repos) if any([repo in repos for repo in _repos]): package = dirpath.strip("/").split("/")[-1] # eg: master/openEuler:Mainline/zip/_services index += 1 - logger.info("{} {}...ok".format(index, _service)) - logger.info("package: {}, repos: {}".format(package, _repos)) + logger.info("%s %s...ok", index, _service) + logger.info("package: %s, repos: %s", package, _repos) for repo in _repos: self._package_repo[package].add(repo) self._repo_package[repo].add(package) diff --git a/src/jobs/repo_mapping.py b/src/jobs/repo_mapping.py index 49b1765..318f2b2 100755 --- a/src/jobs/repo_mapping.py +++ b/src/jobs/repo_mapping.py @@ -38,9 +38,9 @@ class RepoMapping(object): self._exclude_repos = kwargs.get("exclude_jobs") if kwargs.get("exclude_jobs") else [] self._repo_mapping = {} # 保存结果 self._ignored_repos = self._load_ignore_repo(ignored_repos_path, ignored_repos_key) - logger.debug("ignored repos: {}".format(self._ignored_repos)) + logger.debug("ignored repos: %s", self._ignored_repos) self._community_repos = self._load_community_repo(community_path) # 社区repos - logger.debug("community repos: {}".format(self._community_repos)) + logger.debug("community repos: %s", self._community_repos) @staticmethod def _load_ignore_repo(conf_file, ignored_repos_key): @@ -55,7 +55,7 @@ class RepoMapping(object): handler = yaml.safe_load(f) return handler.get(ignored_repos_key, []) except IOError as e: - logger.warning("{} not exist".format(conf_file)) + logger.warning("%s not exist", conf_file) return [] @staticmethod @@ -69,10 +69,10 @@ class RepoMapping(object): with open(community_path, "r") as f: handler = yaml.safe_load(f) repos = {item["name"]: item["type"] for item in handler["repositories"]} - logger.info("repos from community: {}".format(len(repos))) + logger.info("repos from community: %s", len(repos)) return repos except IOError as e: - logger.warning("{} not exist".format(community_path)) + logger.warning("%s not exist", community_path) return [] def _is_valid_repo(self, repo): diff --git a/src/proxy/es_proxy.py b/src/proxy/es_proxy.py index b439951..3b18141 100644 --- a/src/proxy/es_proxy.py +++ b/src/proxy/es_proxy.py @@ -46,10 +46,10 @@ class ESProxy(object): :return: """ try: - logger.debug("es insert: {}".format(body)) + logger.debug("es insert: %s", body) rs = self._es.index(index, body=body) - logger.debug("insert result: {}".format(rs)) + logger.debug("insert result: %s", rs) return rs["result"] == "created" except elasticsearch.ElasticsearchException: logger.exception("elastic search insert document exception") @@ -62,9 +62,9 @@ class ESProxy(object): :param body: :return: """ - logger.debug("es search: {}".format(body)) + logger.debug("es search: %s", body) rs = self._es.search(index=index, body=body) - logger.debug("result: {}".format(rs)) + logger.debug("result: %s", rs) return rs['hits']['hits'] @@ -89,9 +89,9 @@ class ESProxy(object): """ try: body = {"query": query, "script": script} - logger.debug("es update: {}".format(body)) + logger.debug("es update: %s", body) rs = self._es.update_by_query(index, body=body) - logger.debug("update result: {}".format(rs)) + logger.debug("update result: %s", rs) return True except elasticsearch.ElasticsearchException: diff --git a/src/proxy/git_proxy.py b/src/proxy/git_proxy.py index 87fe232..a7bca0b 100644 --- a/src/proxy/git_proxy.py +++ b/src/proxy/git_proxy.py @@ -50,7 +50,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(init_cmd) if ret: - logger.warning("init repository failed, {}".format(ret)) + logger.warning("init repository failed, %s", ret) return None return cls(repo_dir) @@ -70,8 +70,8 @@ class GitProxy(object): self._repo_dir, "--progress" if progress else "", depth, url, pull_request, pull_request) ret, out, _ = shell_cmd_live(fetch_cmd, cap_out=True, cmd_verbose=False) if ret: - logger.error("git fetch failed, {}".format(ret)) - logger.error("{}".format(out)) + logger.error("git fetch failed, %s", ret) + logger.error("%s", out) return False return True @@ -86,7 +86,7 @@ class GitProxy(object): get_content_cmd = "cd {}; git show {}:{}".format(self._repo_dir, commit, file_path) ret, out, _ = shell_cmd_live(get_content_cmd, cap_out=True) if ret: - logger.warning("get file content of commit failed, {}".format(ret)) + logger.warning("get file content of commit failed, %s", ret) return None f = StringIO() @@ -106,7 +106,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(diff_files_cmd, cap_out=True) if ret: - logger.error("get diff files of commits failed, {}".format(ret)) + logger.error("get diff files of commits failed, %s", ret) return [] return out @@ -121,7 +121,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(extract_file_cmd, cap_out=True) if ret: - logger.error("extract diff files of patch failed, {}".format(ret)) + logger.error("extract diff files of patch failed, %s", ret) return [] return [line.split()[-1] for line in out] @@ -137,7 +137,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(apply_patch_cmd) if ret: - #logger.error("apply patch failed, {}".format(ret)) + #logger.error("apply patch failed, %s", ret) return False return True @@ -156,7 +156,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(apply_patch_cmd) if ret: - #logger.error("apply patch failed, {}".format(ret)) + #logger.error("apply patch failed, %s", ret) return False return True @@ -171,7 +171,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(get_commit_cmd, cap_out=True) if ret: - logger.error("get commit id of index failed, {}".format(ret)) + logger.error("get commit id of index failed, %s", ret) return None return out[0] @@ -186,7 +186,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(checkout_cmd) if ret: - logger.warning("checkout failed, {}".format(ret)) + logger.warning("checkout failed, %s", ret) return False return True @@ -201,7 +201,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(checkout_cmd) if ret: - logger.warning("checkout failed, {}".format(ret)) + logger.warning("checkout failed, %s", ret) return False return True @@ -224,7 +224,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(tree_hashes_cmd, cap_out=True) if ret: - logger.error("get tree hashes failed, {}".format(ret)) + logger.error("get tree hashes failed, %s", ret) return None return out @@ -243,7 +243,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(fetch_cmd) if ret: - logger.error("fetch failed, {}".format(ret)) + logger.error("fetch failed, %s", ret) return False return True diff --git a/src/proxy/gitee_proxy.py b/src/proxy/gitee_proxy.py index 3e30c60..3cfa96f 100644 --- a/src/proxy/gitee_proxy.py +++ b/src/proxy/gitee_proxy.py @@ -35,7 +35,7 @@ class GiteeProxy(object): :param comment: 评论内容 :return: 0成功，其它失败 """ - logger.debug("comment pull request {}".format(pr)) + logger.debug("comment pull request %s", pr) comment_pr_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/comments".format(self._owner, self._repo, pr) data = {"access_token": self._token, "body": comment} @@ -58,7 +58,7 @@ class GiteeProxy(object): logger.debug("create tags, but no tags") return True - logger.debug("create tags {} of pull request {}".format(tags, pr)) + logger.debug("create tags %s of pull request %s", tags, pr) pr_tag_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/labels?access_token={}".format( self._owner, self._repo, pr, self._token) @@ -81,7 +81,7 @@ class GiteeProxy(object): logger.debug("replace tags, but no tags") return True - logger.debug("replace all tags with {} of pull request {}".format(tags, pr)) + logger.debug("replace all tags with %s of pull request %s", tags, pr) pr_tag_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/labels?access_token={}".format( self._owner, self._repo, pr, self._token) @@ -99,7 +99,7 @@ class GiteeProxy(object): :param tag: 标签 :return: 0成功，其它失败 """ - logger.debug("delete tag {} of pull request {}".format(tag, pr)) + logger.debug("delete tag %s of pull request %s", tag, pr) pr_tag_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/labels/{}?access_token={}".format( self._owner, self._repo, pr, tag, self._token) @@ -128,7 +128,7 @@ class GiteeProxy(object): """ handler = yaml.safe_load(response.text) repos.update({item["name"]: item["type"] for item in handler["repositories"]}) - logger.info("repos from community: {}".format(len(repos))) + logger.info("repos from community: %s", len(repos)) community_repo_url = "https://gitee.com/openeuler/community/raw/master/repository/src-openeuler.yaml" logger.info("requests repos from community, this will take multi seconds") @@ -143,7 +143,7 @@ class GiteeProxy(object): :param state: pr状态 :return: str or None """ - logger.debug("get last pull request committer, branch: {}, state: {}".format(branch, state)) + logger.debug("get last pull request committer, branch: %s, state: %s", branch, state) pr_url = "https://gitee.com/api/v5/repos/{}/{}/pulls?access_token={}&state={}&base={}" \ "&page=1&per_page=1".format(self._owner, self._repo, self._token, state, branch) @@ -161,7 +161,7 @@ class GiteeProxy(object): if handler: try: committer[0] = handler[0]["user"]["login"] - logger.debug("get last pr committer: {}".format(committer)) + logger.debug("get last pr committer: %s", committer) except KeyError: logger.exception("extract committer info from gitee exception") diff --git a/src/proxy/jenkins_proxy.py b/src/proxy/jenkins_proxy.py index b4b035b..ea40dfc 100644 --- a/src/proxy/jenkins_proxy.py +++ b/src/proxy/jenkins_proxy.py @@ -55,7 +55,7 @@ class JenkinsProxy(object): self._jenkins.create_job(job, config) return True except Exception as e: - logger.exception("create job exception, {}".format(e)) + logger.exception("create job exception, %s", e) return False def update_job(self, job, config): @@ -70,7 +70,7 @@ class JenkinsProxy(object): jks_job.update_config(config) return True except Exception as e: - logger.exception("update job exception, {}".format(e)) + logger.exception("update job exception, %s", e) return False def get_config(self, job): @@ -82,7 +82,7 @@ class JenkinsProxy(object): try: return self._jenkins[job].get_config() except Exception as e: - logger.exception("get config exception, {}".format(e)) + logger.exception("get config exception, %s", e) return None def get_build(self, job, build_no): @@ -95,7 +95,7 @@ class JenkinsProxy(object): try: return self._jenkins[job].get_build(build_no) except Exception as e: - logger.exception("get job build exception, {}".format(e)) + logger.exception("get job build exception, %s", e) return None @classmethod @@ -109,7 +109,7 @@ class JenkinsProxy(object): parent_build = build.get_upstream_build() return parent_build.get_upstream_build() if parent_build else None except Exception as e: - logger.exception("get grandpa build exception, {}".format(e)) + logger.exception("get grandpa build exception, %s", e) return None def _get_upstream_jobs(self, job): @@ -119,22 +119,22 @@ class JenkinsProxy(object): :param job: Jenkins Job object :return: """ - logger.debug("get upstream jobs of {}".format(job._data["fullName"])) + logger.debug("get upstream jobs of %s", job._data["fullName"]) jobs = [] for project in job._data["upstreamProjects"]: # but is the only way of get upstream projects info url = project.get("url") name = project.get("name") - logger.debug("upstream project: {} {}".format(url, name)) + logger.debug("upstream project: %s %s", url, name) m = re.match("(.*)/job/.*", url) # remove last part of job url, greedy match base_url = m.group(1) - logger.debug("base url {}".format(base_url)) + logger.debug("base url %s", base_url) try: j = jenkins.Jenkins(base_url, self._username, self._token, timeout=self._timeout) jobs.append(j[name]) except Exception as e: - logger.exception("get job of {} exception".format(url)) + logger.exception("get job of %s exception", url) continue return jobs @@ -167,17 +167,17 @@ class JenkinsProxy(object): cause_build = cause_job.get_build(cause_build_id) cause_cause_build_id = cause_build.get_upstream_build_number() - logger.debug("cause_build_id: {}, cause_job_name: {}, cause_cause_build_id: {}".format( - cause_build_id, cause_job_name, cause_cause_build_id)) + logger.debug("cause_build_id: %s, cause_job_name: %s, cause_cause_build_id: %s", + cause_build_id, cause_job_name, cause_cause_build_id) upstream_builds = [] for upstream_job in upstream_jobs: - logger.debug("{}".format(upstream_job._data["fullName"])) + logger.debug("%s", upstream_job._data["fullName"]) for build_id in upstream_job.get_build_ids(): - logger.debug("try build id {}".format(build_id)) + logger.debug("try build id %s", build_id) a_build = upstream_job.get_build(build_id) if a_build.get_upstream_build_number() == cause_cause_build_id: - logger.debug("build id {} match".format(build_id)) + logger.debug("build id %s match", build_id) upstream_builds.append(a_build) break diff --git a/src/proxy/kafka_proxy.py b/src/proxy/kafka_proxy.py index c7dcde4..110b46e 100644 --- a/src/proxy/kafka_proxy.py +++ b/src/proxy/kafka_proxy.py @@ -49,12 +49,12 @@ class KafkaProducerProxy(object): :return: """ try: - logger.debug("kafka send: {}, {}".format(key, value)) + logger.debug("kafka send: %s, %s", key, value) future = self._kp.send(topic, value=value, key=key) rs = future.get(timeout=self._timeout) - logger.debug("kafka send result: {}".format(rs)) + logger.debug("kafka send result: %s", rs) return True except errors.KafkaTimeoutError: logger.exception("kafka send timeout exception") diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index 6724748..98fd978 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -45,7 +45,7 @@ class OBSProxy(object): cmd = "osc ll {} {}".format(project, package) ret, rs, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.error("list project package error, {}".format(ret)) + logger.error("list project package error, %s", ret) return None return rs @@ -62,7 +62,7 @@ class OBSProxy(object): "--show-exclude" if show_exclude else "", project, package, arch) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.debug("list obs repos of arch error, {}".format(ret)) + logger.debug("list obs repos of arch error, %s", ret) return [] rs = [] @@ -88,7 +88,7 @@ class OBSProxy(object): cmd = "osc results {} --csv |grep {} | awk -F';' '{{print $1}}'".format(project, state) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.debug("list package of state error, {}".format(ret)) + logger.debug("list package of state error, %s", ret) return [] return out @@ -105,23 +105,24 @@ class OBSProxy(object): _ = os.path.isdir(project) and shutil.rmtree(project) cmd = "osc co {} {}".format(project, package) - logger.info("osc co {} {}".format(project, package)) + logger.info("osc co %s %s", project, package) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: - logger.error("checkout package error, {}".format(ret)) + logger.error("checkout package error, %s", ret) return False return True @staticmethod - def build_package(project, package, repo, arch, mpac, debug=False, root_build=False, disable_cpio=False): + def build_package(project, package, repo, arch, spec, mpac, debug=False, root_build=False, disable_cpio=False): """ build :param project: :param package: :param repo: :param arch: + :param spec: :param mpac: multibuild package :param debug: :return: @@ -130,15 +131,15 @@ class OBSProxy(object): root_opt = "--userootforbuild" if root_build else "" debuginfo_opt = "--disable-debuginfo" if not debug else "" disable_cpio_bulk = "--disable-cpio-bulk-download" if disable_cpio else "" - cmd = "cd {}; osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( - package_path, repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) + cmd = "cd {}; osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + package_path, repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) - logger.info("osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( - repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac)) + logger.info("osc build %s %s %s %s %s %s --no-verify --clean --noservice -M %s", + repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: - logger.error("build package error, {}".format(ret)) + logger.error("build package error, %s", ret) return False return True @@ -156,7 +157,7 @@ class OBSProxy(object): cmd = "osc api /build/{}/{}/{}/{}/_history".format(project, repo, arch, package) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.debug("list build history of package error, {}".format(ret)) + logger.debug("list build history of package error, %s", ret) return "" return "\n".join(out) diff --git a/src/proxy/requests_proxy.py b/src/proxy/requests_proxy.py index b88129f..87de829 100644 --- a/src/proxy/requests_proxy.py +++ b/src/proxy/requests_proxy.py @@ -38,9 +38,9 @@ def do_requests(method, url, querystring=None, body=None, auth=None, timeout=30, :return: """ try: - logger.debug("http requests, {} {} {}".format(method, url, timeout)) - logger.debug("querystring: {}".format(querystring)) - logger.debug("body: {}".format(body)) + logger.debug("http requests, %s %s %s", method, url, timeout) + logger.debug("querystring: %s", querystring) + logger.debug("body: %s", body) if method.lower() not in ["get", "post", "put", "delete"]: return -1 @@ -60,7 +60,7 @@ def do_requests(method, url, querystring=None, body=None, auth=None, timeout=30, else: rs = func(url, timeout=timeout) - logger.debug("status_code {}".format(rs.status_code)) + logger.debug("status_code %s", rs.status_code) if rs.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.no_content]: return 1 @@ -78,11 +78,11 @@ def do_requests(method, url, querystring=None, body=None, auth=None, timeout=30, return 0 except requests.exceptions.SSLError as e: - logger.warning("requests {} ssl exception, {}".format(url, e)) + logger.warning("requests %s ssl exception, %s", url, e) return -2 except requests.exceptions.Timeout as e: logger.warning("requests timeout") return 2 except requests.exceptions.RequestException as e: - logger.warning("requests exception, {}".format(e)) + logger.warning("requests exception, %s", e) return 3 diff --git a/src/requirements b/src/requirements index d3fd573..7d34070 100644 --- a/src/requirements +++ b/src/requirements @@ -11,4 +11,4 @@ chardet kafka-python elasticsearch retrying -scanoss-scanner +scanoss diff --git a/src/tools/obs_package_build_history.py b/src/tools/obs_package_build_history.py index cd08637..ba9a8d0 100644 --- a/src/tools/obs_package_build_history.py +++ b/src/tools/obs_package_build_history.py @@ -46,7 +46,7 @@ class JobBuildHistory(object): try: root = ET.fromstring(history) except ParseError: - logger.exception("package: {}, build history: {}".format(package, history)) + logger.exception("package: %s, build history: %s", package, history) return {"package": package, "max": 0, "min": 0, "average": 0, "times": -1} duration = [int(ele.get("duration")) for ele in root.findall("entry")] @@ -74,11 +74,11 @@ class JobBuildHistory(object): for index in range(batch): works = [gevent.spawn(JobBuildHistory.get_package_job_duration, project, package, repo, arch) for package in packages[index * concurrency: (index + 1) * concurrency]] - logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) + logger.info("%s works, %s/%s ", len(works), index + 1, batch) gevent.joinall(works) for work in works: - logger.debug("{}: {}".format(work.value["package"], work.value)) - logger.info("{} ...done".format(work.value["package"])) + logger.debug("%s: %s", work.value["package"], work.value) + logger.info("{%s} ...done", work.value["package"]) rs.append(work.value) time.sleep(1) diff --git a/src/tools/obs_package_build_report.py b/src/tools/obs_package_build_report.py index cdb5084..d5cf53f 100644 --- a/src/tools/obs_package_build_report.py +++ b/src/tools/obs_package_build_report.py @@ -97,17 +97,17 @@ class ObsPackageBuildReport(object): # try: # branch = self.__class__.PROJECT_BRANCH_MAPPING[self._project] # except KeyError: -# logger.exception("project {} not support".format(self._project)) +# logger.exception("project %s not support", self._project) # return branch = "master" # get packages in project of state packages = OBSProxy.list_packages_of_state(self._project, self._state) - logger.info("project {} state {}, find {} packages".format(self._project, self._state, len(packages))) + logger.info("project %s state %s, find %s packages", self._project, self._state, len(packages)) # get last pr committer for index, package in enumerate(packages): - logger.info("{}: {}".format(index, package)) + logger.info("%s: %s", index, package) gp = GiteeProxy(self.GITEE_OWNER, package, gitee_api_token) committer = gp.get_last_pr_committer(branch) real_name = self._real_name_mapping.get(committer, "N/A") diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index c68c65c..e3be542 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -8,12 +8,18 @@ no_fmt_logger = logging.getLogger("no_fmt") def shell_cmd(cmd, inmsg=None): - logger.debug("exec cmd -- [{}]".format(cmd)) + """ + 创建子进程执行命令，返回执行结果 + :param cmd: 命令 + :param inmsg: 输入 + :return: + """ + logger.debug("exec cmd -- [%s]", cmd) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if inmsg: p.stdin.write(inmsg) out, err = p.communicate() - logger.debug("iret: {}, rs: {}, err: {}".format(p.returncode, out, err)) + logger.debug("iret: %s, rs: %s, err: %s", p.returncode, out, err) return p.returncode, out, err @@ -29,7 +35,7 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False :return: """ if cmd_verbose: - logger.debug("exec cmd -- {}".format(cmd)) + logger.debug("exec cmd -- %s", cmd) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if cap_in: @@ -51,13 +57,13 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False break if cap_out: - logger.debug("total {} lines output".format(len(out))) + logger.debug("total %s lines output", len(out)) ret = p.poll() err = None if ret: - logger.debug("return code {}".format(ret)) + logger.debug("return code %s", ret) while True: line= p.stderr.readline() if not line: diff --git a/test/ac/acl/package_yaml/test_check_repo.py b/test/ac/acl/package_yaml/test_check_repo.py index 8bf40f2..f1d7522 100644 --- a/test/ac/acl/package_yaml/test_check_repo.py +++ b/test/ac/acl/package_yaml/test_check_repo.py @@ -51,9 +51,9 @@ class TestGetReleaseTags(unittest.TestCase): with open(filepath, 'r') as yaml_data: # load yaml data result = yaml.safe_load(yaml_data) except IOError as e: - logging.warning("package yaml not exist. {}".format(str(e))) + logging.warning("package yaml not exist. %s", str(e)) except yaml.YAMLError as exc: - logging.warning("Error parsering YAML: {}".format(str(exc))) + logging.warning("Error parsering YAML: %s", str(exc)) finally: return result -- Gitee From c3ddb758dd30ece365211330015eaf78b6417c9e Mon Sep 17 00:00:00 2001 From: wanghuan158 Date: Tue, 23 Nov 2021 09:07:29 +0000 Subject: [PATCH 33/42] update src/build/gitee_comment.py. --- src/build/gitee_comment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index e6e94fe..5d7e412 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -103,7 +103,7 @@ class Comment(object): comments.extend(self._comment_of_ac(self._up_up_builds[0])) if self._up_builds: comments.extend(self._comment_of_build(self._up_builds)) - comments.extend(self._comment_of_check_abi(self._up_builds)) + #comments.extend(self._comment_of_check_abi(self._up_builds)) comments.extend(self._comment_of_compare_package(self._up_builds)) comments.append("") @@ -201,7 +201,7 @@ class Comment(object): for item in content: ac_result = ACResult.get_instance(item.get("result")) comments.append(self.__class__.comment_html_table_tr( - "compare_package", ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) + item.get("name"), ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) break logger.info("compare package comment: {}".format(comments)) -- Gitee From a6a49bbb995d146bf855c23f04857967417e8a50 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Tue, 23 Nov 2021 17:49:18 +0800 Subject: [PATCH 34/42] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E6=97=A5=E5=BF=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/gitee_comment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 5d7e412..719791c 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -195,10 +195,13 @@ class Comment(object): with open(check_abi_comment_file, "r") as f: try: content = yaml.safe_load(f) + logging.info("content = %s", content) + except YAMLError: # yaml base exception logger.exception("illegal yaml format of check abi comment file ") logger.debug("comment: {}".format(content)) for item in content: + logging.info("item_name = %s", item.get("name")) ac_result = ACResult.get_instance(item.get("result")) comments.append(self.__class__.comment_html_table_tr( item.get("name"), ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) -- Gitee From 7645b33883cbec9e186b893eafed0d070509f3a8 Mon Sep 17 00:00:00 2001 From: wanghuan158 Date: Wed, 24 Nov 2021 02:43:35 +0000 Subject: [PATCH 35/42] update src/build/extra_work.py. --- src/build/extra_work.py | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index b558474..4403534 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -162,6 +162,12 @@ class ExtraWork(object): logger.exception("save check abi comment exception") def get_dict(self, key_list, data): + """ + 获取字典value + :param key_list: key列表 + :param data: 字典 + :return: + """ if not isinstance(data, dict): return None for key, value in data.items(): @@ -175,6 +181,13 @@ class ExtraWork(object): @staticmethod def get_item_data(item, data, fp): + """ + 输出diff的详情至文件fp中 + :param item: + :param data: + :param fp: + :return: + """ str_list = [] if isinstance(data, list): str_list.append(" {}:\n".format(item)) @@ -192,6 +205,12 @@ class ExtraWork(object): fp.writelines(str_list) def show_rpm_diff(self, compare_details, fp): + """ + 输出rpm包差异 + :param compare_details:差异详情 + :param fp: 存放展示结果的文件句柄 + :return: + """ for key in ["more", "less", "diff"]: key_list = [key, "%s_details" % key] details = self.get_dict(key_list, compare_details) @@ -208,6 +227,13 @@ class ExtraWork(object): fp.write("%s\n" % rpm) def output_result_to_console(self, json_file, pr_link, fp): + """ + 解析结果文件并输出展示到jenkins上 + :param json_file: 结果文件json + :param pr_link: pr_link + :param fp: 存放展示结果的文件句柄 + :return: 比对结果 + """ all_data = {} diff_item = ["more", "less", "diff"] @@ -265,7 +291,7 @@ class ExtraWork(object): with open(tmp_file, "r") as fp: logger.info(fp.read()) - comment = {"name": "compare/{}/{}".format(package_arch, self._repo), "result": result, + comment = {"name": "compare_package/{}/{}".format(package_arch, self._repo), "result": result, "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, os.path.basename(output), package_url)} -- Gitee From 9cf32c7951a82416b11f4757acbd2153adc5be84 Mon Sep 17 00:00:00 2001 From: wanghuan158 Date: Wed, 24 Nov 2021 02:44:35 +0000 Subject: [PATCH 36/42] update src/build/gitee_comment.py. --- src/build/gitee_comment.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 719791c..5d7e412 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -195,13 +195,10 @@ class Comment(object): with open(check_abi_comment_file, "r") as f: try: content = yaml.safe_load(f) - logging.info("content = %s", content) - except YAMLError: # yaml base exception logger.exception("illegal yaml format of check abi comment file ") logger.debug("comment: {}".format(content)) for item in content: - logging.info("item_name = %s", item.get("name")) ac_result = ACResult.get_instance(item.get("result")) comments.append(self.__class__.comment_html_table_tr( item.get("name"), ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) -- Gitee From 23582748104e978ad5dd2bb12781eb921f99bc3e Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 24 Nov 2021 14:21:48 +0800 Subject: [PATCH 37/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E5=90=84=E5=88=86=E6=94=AFpr=E6=95=B0=E7=94=9F=E6=88=90csv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/extra_work.py | 63 ++++++++++++++++++++++++++++++++++++++ src/build/gitee_comment.py | 7 +++-- 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index f6c2cb0..8db40e6 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -22,6 +22,7 @@ import logging.config import logging import yaml import json +import datetime from src.build.obs_repo_source import OBSRepoSource @@ -318,6 +319,58 @@ class ExtraWork(object): yaml.safe_dump(comments, f) # list except IOError: logger.exception("save compare package comment exception") + def stats_file_read(self, stat_path) : + """ + 读取二进制变更检查统计表 + :param stat_path: 二进制文件及差异报告文档路径 + :return: + """ + listOfLines = [] + if not os.path.exists(stat_path): + listOfLines.append(["分支名", "仓库名", "架构", "二进制变更次数", "归档路径", "最新合入时间"]) + self.stats_file_write(listOfLines, stat_path) + with open(stat_path, "r", encoding='utf-8') as fileHandler: + listOfLines = fileHandler.read().splitlines() + for index in range(len(listOfLines)): + print(listOfLines[index]) + listOfLines[index] = listOfLines[index].split(",") + return listOfLines + + def stats_file_write(self, listOfLines, stat_path): + """ + 写回二进制变更检查统计表 + :param listOfLines: 统计内容 + :param stat_path: 二进制文件及差异报告文档路径 + :return: + """ + with open(stat_path, "w", encoding='utf-8') as fileHandler: + for index in range(len(listOfLines)): + fileHandler.write(",".join(listOfLines[index]) + "\n") + + def stats_compare_package(self, branch_name, repo, arch, repo_archive_url, stat_path): + """ + 更新二进制变更检查统计表 + :param branch_name: 分支名 + :param repo: 仓库名 + :param arch: cpu架构 + :param repo_archive_url: 二进制文件及差异报告文档路径 + :param stat_path: 二进制文件及差异报告文档路径 + :return: + """ + listOfLines = self.stats_file_read(stat_path) + find = -1 + for index in range(len(listOfLines)): + if listOfLines[index][0] == branch_name and listOfLines[index][1] == repo and listOfLines[index][2] == arch: + find = index + now = datetime.datetime.now().strftime("%Y-%b-%d %H:%M") + if find != -1: + count = int(listOfLines[find][3]) + listOfLines[find][3] = str(count + 1) + listOfLines[find][4] = repo_archive_url + listOfLines[find][5] = now + else: + listOfLines.append([branch_name, repo, arch, str(1), repo_archive_url, now]) + self.stats_file_write(listOfLines, stat_path) def check_install_rpm(self, branch_name, arch, install_root): """ @@ -379,6 +432,8 @@ def checkabi(args, ew): def comparepackage(args, ew): ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, args.json_path, args.pr_link) +def statsdiff(args, ew): + ew.stats_compare_package(args.branch, args.repo, args.arch, args.repo_archive_url, args.stat_path) def checkinstall(args, ew): ew.check_install_rpm(args.branch_name, args.arch, args.install_root) @@ -425,6 +480,14 @@ if "__main__" == __name__: parser_comparepackage.add_argument("-p", "--package", type=str, help="obs package") parser_comparepackage.set_defaults(func=comparepackage) + #添加子命令 statsdiff + parser_statsdiff = subparsers.add_parser('statsdiff', help='add help') + parser_statsdiff.add_argument("-r", type=str, dest="repo", help="package") + parser_statsdiff.add_argument("-b", type=str, dest="branch", help="branch") + parser_statsdiff.add_argument("-a", type=str, dest="arch", help="arch") + parser_statsdiff.add_argument("-u", type=str, dest="repo_archive_url", help="repo archive url") + parser_statsdiff.add_argument("-p", type=str, dest="stat_path", help="stat_path") + parser_statsdiff.set_defaults(func=statsdiff) #添加子命令 checkinstall parser_checkinstall = subparsers.add_parser('checkinstall', help='add help') parser_checkinstall.add_argument("-r", type=str, dest="branch_name", help="obs project name") diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 5d778bc..cf9bd94 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -199,9 +199,12 @@ class Comment(object): logger.debug("comment: %s", content) for item in content: ac_result = ACResult.get_instance(item.get("result")) + # comments.append(self.__class__.comment_html_table_tr( + # item.get("name"), ac_result.emoji, ac_result.hint, item.get("link", ""), + # "markdown" if "link" in item else "", hashtag=False)) comments.append(self.__class__.comment_html_table_tr( - item.get("name"), ac_result.emoji, ac_result.hint, item.get("link", ""), - "markdown" if "link" in item else "", hashtag=False)) + item.get("name"), ac_result.emoji, ac_result.hint, + "{}{}".format(build.get_build_url(), "console"), build.buildno, hashtag=False)) break logger.info("check abi comment: %s", comments) -- Gitee From 70b025c03b55206dd34c5135254188568654bc47 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 24 Nov 2021 14:37:41 +0800 Subject: [PATCH 38/42] =?UTF-8?q?=E4=BF=AE=E6=94=B9csv=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E7=94=A8excel=E6=89=93=E5=BC=80=E5=90=8E=E4=B9=B1=E7=A0=81?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/extra_work.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 8db40e6..b1b31ba 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -319,7 +319,8 @@ class ExtraWork(object): yaml.safe_dump(comments, f) # list except IOError: logger.exception("save compare package comment exception") - def stats_file_read(self, stat_path) : + + def stats_file_read(self, stat_path): """ 读取二进制变更检查统计表 :param stat_path: 二进制文件及差异报告文档路径 @@ -329,7 +330,7 @@ class ExtraWork(object): if not os.path.exists(stat_path): listOfLines.append(["分支名", "仓库名", "架构", "二进制变更次数", "归档路径", "最新合入时间"]) self.stats_file_write(listOfLines, stat_path) - with open(stat_path, "r", encoding='utf-8') as fileHandler: + with open(stat_path, "r") as fileHandler: listOfLines = fileHandler.read().splitlines() for index in range(len(listOfLines)): print(listOfLines[index]) @@ -343,7 +344,7 @@ class ExtraWork(object): :param stat_path: 二进制文件及差异报告文档路径 :return: """ - with open(stat_path, "w", encoding='utf-8') as fileHandler: + with open(stat_path, "w") as fileHandler: for index in range(len(listOfLines)): fileHandler.write(",".join(listOfLines[index]) + "\n") -- Gitee From 526c6583ed00979c6197ecf16fa561e01d5b3351 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 24 Nov 2021 14:42:33 +0800 Subject: [PATCH 39/42] =?UTF-8?q?comment=E4=B8=AD=E9=93=BE=E6=8E=A5?= =?UTF-8?q?=E5=8A=A0=E4=B8=8A#=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/gitee_comment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index cf9bd94..1b7840a 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -204,7 +204,7 @@ class Comment(object): # "markdown" if "link" in item else "", hashtag=False)) comments.append(self.__class__.comment_html_table_tr( item.get("name"), ac_result.emoji, ac_result.hint, - "{}{}".format(build.get_build_url(), "console"), build.buildno, hashtag=False)) + "{}{}".format(build.get_build_url(), "console"), build.buildno)) break logger.info("check abi comment: %s", comments) -- Gitee From 90b6b62a1038336603f581e1708eb928c2125d70 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 24 Nov 2021 15:43:33 +0800 Subject: [PATCH 40/42] =?UTF-8?q?=E6=97=A5=E5=BF=97=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/extra_work.py | 64 +++++++++++++++++++++-------------------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index b1b31ba..41993ad 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -296,23 +296,18 @@ class ExtraWork(object): "link": self._rpm_package.checkabi_md_in_repo(committer, self._repo, package_arch, os.path.basename(output), package_url)} - logger.debug("compare package comment: {}".format(comment)) - try: - with open(comment_file, "r") as f: # one repo with multi build package - comments = yaml.safe_load(f) - except IOError as e: - logger.debug("no history compare package comment") + logger.debug("compare package comment: %s", comment) comments = [] if os.path.exists(comment_file): try: with open(comment_file, "r") as f: # one repo with multi build package comments = yaml.safe_load(f) - except IOError: - logger.exception("yaml load compare package comment file exception") + except IOError as err: + logger.exception("yaml load compare package comment file exception:%s", err) comments.append(comment) - logger.debug("compare package comments: {}".format(comments)) + logger.debug("compare package comments: %s", comments) logging.info("compare package finish") try: with open(comment_file, "w") as f: @@ -322,9 +317,9 @@ class ExtraWork(object): def stats_file_read(self, stat_path): """ - 读取二进制变更检查统计表 - :param stat_path: 二进制文件及差异报告文档路径 - :return: + 读取二进制变更检查统计表 + :param stat_path: 二进制文件及差异报告文档路径 + :return: """ listOfLines = [] if not os.path.exists(stat_path): @@ -337,12 +332,13 @@ class ExtraWork(object): listOfLines[index] = listOfLines[index].split(",") return listOfLines - def stats_file_write(self, listOfLines, stat_path): + @staticmethod + def stats_file_write(listOfLines, stat_path): """ - 写回二进制变更检查统计表 - :param listOfLines: 统计内容 - :param stat_path: 二进制文件及差异报告文档路径 - :return: + 写回二进制变更检查统计表 + :param listOfLines: 统计内容 + :param stat_path: 二进制文件及差异报告文档路径 + :return: """ with open(stat_path, "w") as fileHandler: for index in range(len(listOfLines)): @@ -350,13 +346,13 @@ class ExtraWork(object): def stats_compare_package(self, branch_name, repo, arch, repo_archive_url, stat_path): """ - 更新二进制变更检查统计表 - :param branch_name: 分支名 - :param repo: 仓库名 - :param arch: cpu架构 - :param repo_archive_url: 二进制文件及差异报告文档路径 - :param stat_path: 二进制文件及差异报告文档路径 - :return: + 更新二进制变更检查统计表 + :param branch_name: 分支名 + :param repo: 仓库名 + :param arch: cpu架构 + :param repo_archive_url: 二进制文件及差异报告文档路径 + :param stat_path: 二进制文件及差异报告文档路径 + :return: """ listOfLines = self.stats_file_read(stat_path) find = -1 @@ -387,8 +383,6 @@ class ExtraWork(object): _ = not os.path.exists(install_root) and os.mkdir(install_root) logger.info("create install root directory: %s", install_root) - repo_name_prefix = "check_install" - # 2. prepare repo repo_source = OBSRepoSource("http://119.3.219.20:82") # obs 实时构建repo地址 repo_config = repo_source.generate_repo_info(branch_name, arch, "check_install") @@ -419,26 +413,33 @@ class ExtraWork(object): else: logger.info("install rpm success") + def notify(args, ew): # run after copy rpm to rpm repo if ew.is_pkgship_need_notify(args.pkgship_meta): ew.pkgship_notify( args.notify_url, args.token, args.rpm_repo_url, args.arch, args.notify_user, args.notify_password) + def checkabi(args, ew): # run before copy rpm to rpm repo ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, args.obs_addr, args.branch_name, args.obs_repo_url) + def comparepackage(args, ew): ew.compare_rpm_diff(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, args.json_path, args.pr_link) + + def statsdiff(args, ew): ew.stats_compare_package(args.branch, args.repo, args.arch, args.repo_archive_url, args.stat_path) + def checkinstall(args, ew): ew.check_install_rpm(args.branch_name, args.arch, args.install_root) + if "__main__" == __name__: parser = argparse.ArgumentParser() parser.add_argument("-p", "--package", type=str, default="src-openeuler", help="obs package") @@ -446,7 +447,7 @@ if "__main__" == __name__: default="/home/jenkins/agent/buildroot/home/abuild/rpmbuild", help="rpmbuild dir") subparsers = parser.add_subparsers(help='sub-command help') - #添加子命令 notify + # 添加子命令 notify parser_notify = subparsers.add_parser('notify', help='add help') parser_notify.add_argument("-n", type=str, dest="notify_url", help="target branch that merged to ") parser_notify.add_argument("-t", type=str, dest="token", default=os.getcwd(), help="obs workspace dir path") @@ -456,7 +457,7 @@ if "__main__" == __name__: parser_notify.add_argument("-w", type=str, dest="notify_password", help="notify trigger password") parser_notify.set_defaults(func=notify) - #添加子命令 checkabi + # 添加子命令 checkabi parser_checkabi = subparsers.add_parser('checkabi', help='add help') parser_checkabi.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") parser_checkabi.add_argument("-a", type=str, dest="arch", help="build arch") @@ -469,7 +470,7 @@ if "__main__" == __name__: parser_checkabi.add_argument("-e", type=str, dest="comment_file", help="compare package result comment") parser_checkabi.set_defaults(func=checkabi) - #添加子命令 comparepackage + # 添加子命令 comparepackage parser_comparepackage = subparsers.add_parser('comparepackage', help='add help') parser_comparepackage.add_argument("-l", type=str, dest="rpm_repo_url", help="rpm repo where rpm saved") parser_comparepackage.add_argument("-a", type=str, dest="arch", help="build arch") @@ -481,7 +482,7 @@ if "__main__" == __name__: parser_comparepackage.add_argument("-p", "--package", type=str, help="obs package") parser_comparepackage.set_defaults(func=comparepackage) - #添加子命令 statsdiff + # 添加子命令 statsdiff parser_statsdiff = subparsers.add_parser('statsdiff', help='add help') parser_statsdiff.add_argument("-r", type=str, dest="repo", help="package") parser_statsdiff.add_argument("-b", type=str, dest="branch", help="branch") @@ -489,7 +490,8 @@ if "__main__" == __name__: parser_statsdiff.add_argument("-u", type=str, dest="repo_archive_url", help="repo archive url") parser_statsdiff.add_argument("-p", type=str, dest="stat_path", help="stat_path") parser_statsdiff.set_defaults(func=statsdiff) - #添加子命令 checkinstall + + # 添加子命令 checkinstall parser_checkinstall = subparsers.add_parser('checkinstall', help='add help') parser_checkinstall.add_argument("-r", type=str, dest="branch_name", help="obs project name") parser_checkinstall.add_argument("-a", type=str, dest="arch", help="build arch") -- Gitee From 10e9a8f0d96d8004cb21a21357083a6792d323ba Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 24 Nov 2021 16:39:09 +0800 Subject: [PATCH 41/42] =?UTF-8?q?=E9=9D=99=E6=80=81=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/extra_work.py | 108 ++++++++++++++++++------------------- src/build/gitee_comment.py | 15 +++--- 2 files changed, 60 insertions(+), 63 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 41993ad..50269eb 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -20,14 +20,12 @@ import os import argparse import logging.config import logging -import yaml import json import datetime +import yaml from src.build.obs_repo_source import OBSRepoSource -SUCCESS = "SUCCESS" -FAILED = "FAILED" class ExtraWork(object): """ @@ -49,23 +47,22 @@ class ExtraWork(object): :param pkgship_meta_path: 保存门禁中解析的pkgship spec版本元信息文件路径 :return: """ - if self._repo == "pkgship": # 只有pkgship包需要通知 - try: - with open(pkgship_meta_path, "r") as f: - pkgship_meta = yaml.safe_load(f) - logger.debug("pkgship meta: %s", pkgship_meta) - if pkgship_meta.get("compare_version") == 1: # version upgrade - logger.debug("pkgship: notify") - return True - except IOError: - # file not exist, bug - logger.warning("pkgship meta file not exist!") - return True - - return False + if self._repo != "pkgship": # 只有pkgship包需要通知 + return False + try: + with open(pkgship_meta_path, "r") as f: + pkgship_meta = yaml.safe_load(f) + logger.debug("pkgship meta: %s", pkgship_meta) + if pkgship_meta.get("compare_version") == 1: # version upgrade + logger.debug("pkgship: notify") + return True + except IOError: + # file not exist, bug + logger.warning("pkgship meta file not exist!") + return True - def pkgship_notify( - self, notify_url, notify_token, package_url, package_arch, notify_jenkins_user, notify_jenkins_password): + def pkgship_notify(self, notify_url, notify_token, package_url, package_arch, + notify_jenkins_user, notify_jenkins_password): """ notify :param notify_url: notify url @@ -140,11 +137,6 @@ class ExtraWork(object): comment = {"name": "check_abi/{}/{}".format(package_arch, self._repo), "result": "SUCCESS"} logger.debug("check abi comment: %s", comment) - try: - with open(comment_file, "r") as f: # one repo with multi build package - comments = yaml.safe_load(f) - except IOError as e: - logger.debug("no history check abi comment") comments = [] if os.path.exists(comment_file): @@ -153,6 +145,8 @@ class ExtraWork(object): comments = yaml.safe_load(f) except IOError: logger.exception("yaml load check abi comment file exception") + else: + logger.debug("no history check abi comment") comments.append(comment) logger.debug("check abi comments: %s", comments) @@ -221,7 +215,7 @@ class ExtraWork(object): fp.write("diff[old_rpm_name,new_rpm_name]:\n") for rpm in details: new_rpm_name = self.get_dict([rpm, "name", "new"], details) - fp.write("%s,%s\n" %(rpm, new_rpm_name)) + fp.write("%s,%s\n" % (rpm, new_rpm_name)) else: fp.write("%s:\n" % key) for rpm in details: @@ -240,7 +234,7 @@ class ExtraWork(object): if not os.path.exists(json_file): logger.error("%s not exists", json_file) - return FAILED + return "FAILED" with open(json_file, "r") as data: all_data = json.load(data) # 写入pr link到json文件 @@ -249,9 +243,9 @@ class ExtraWork(object): json.dump(all_data, data) compare_result = all_data.get("compare_result") compare_details = all_data.get("compare_details") - fp.write("compare <%s> package %s\n" %(self._repo, compare_result)) + fp.write("compare <%s> package %s\n" % (self._repo, compare_result)) if not compare_details: - return FAILED + return "FAILED" # 显示所有的差异 self.show_rpm_diff(compare_details, fp) @@ -272,9 +266,9 @@ class ExtraWork(object): self.get_item_data(items[1], items_data, fp) if compare_result == "pass": - return SUCCESS + return "SUCCESS" else: - return FAILED + return "FAILED" def compare_rpm_diff(self, package_url, package_arch, output, committer, comment_file, result_path, pr_link): @@ -305,6 +299,8 @@ class ExtraWork(object): comments = yaml.safe_load(f) except IOError as err: logger.exception("yaml load compare package comment file exception:%s", err) + else: + logger.debug("no history compare package comment") comments.append(comment) logger.debug("compare package comments: %s", comments) @@ -321,28 +317,28 @@ class ExtraWork(object): :param stat_path: 二进制文件及差异报告文档路径 :return: """ - listOfLines = [] + list_of_lines = [] if not os.path.exists(stat_path): - listOfLines.append(["分支名", "仓库名", "架构", "二进制变更次数", "归档路径", "最新合入时间"]) - self.stats_file_write(listOfLines, stat_path) - with open(stat_path, "r") as fileHandler: - listOfLines = fileHandler.read().splitlines() - for index in range(len(listOfLines)): - print(listOfLines[index]) - listOfLines[index] = listOfLines[index].split(",") - return listOfLines + list_of_lines.append(["分支名", "仓库名", "架构", "二进制变更次数", "归档路径", "最新合入时间"]) + self.stats_file_write(list_of_lines, stat_path) + with open(stat_path, "r") as filehandler: + list_of_lines = filehandler.read().splitlines() + for index in range(len(list_of_lines)): + print(list_of_lines[index]) + list_of_lines[index] = list_of_lines[index].split(",") + return list_of_lines @staticmethod - def stats_file_write(listOfLines, stat_path): + def stats_file_write(list_of_lines, stat_path): """ 写回二进制变更检查统计表 - :param listOfLines: 统计内容 + :param list_of_lines: 统计内容 :param stat_path: 二进制文件及差异报告文档路径 :return: """ - with open(stat_path, "w") as fileHandler: - for index in range(len(listOfLines)): - fileHandler.write(",".join(listOfLines[index]) + "\n") + with open(stat_path, "w") as filehandler: + for index in range(len(list_of_lines)): + filehandler.write(",".join(list_of_lines[index]) + "\n") def stats_compare_package(self, branch_name, repo, arch, repo_archive_url, stat_path): """ @@ -354,20 +350,20 @@ class ExtraWork(object): :param stat_path: 二进制文件及差异报告文档路径 :return: """ - listOfLines = self.stats_file_read(stat_path) + list_of_lines = self.stats_file_read(stat_path) find = -1 - for index in range(len(listOfLines)): - if listOfLines[index][0] == branch_name and listOfLines[index][1] == repo and listOfLines[index][2] == arch: + for index in range(len(list_of_lines)): + if list_of_lines[index][0] == branch_name and list_of_lines[index][1] == repo and list_of_lines[index][2] == arch: find = index now = datetime.datetime.now().strftime("%Y-%b-%d %H:%M") if find != -1: - count = int(listOfLines[find][3]) - listOfLines[find][3] = str(count + 1) - listOfLines[find][4] = repo_archive_url - listOfLines[find][5] = now + count = int(list_of_lines[find][3]) + list_of_lines[find][3] = str(count + 1) + list_of_lines[find][4] = repo_archive_url + list_of_lines[find][5] = now else: - listOfLines.append([branch_name, repo, arch, str(1), repo_archive_url, now]) - self.stats_file_write(listOfLines, stat_path) + list_of_lines.append([branch_name, repo, arch, str(1), repo_archive_url, now]) + self.stats_file_write(list_of_lines, stat_path) def check_install_rpm(self, branch_name, arch, install_root): """ @@ -495,13 +491,15 @@ if "__main__" == __name__: parser_checkinstall = subparsers.add_parser('checkinstall', help='add help') parser_checkinstall.add_argument("-r", type=str, dest="branch_name", help="obs project name") parser_checkinstall.add_argument("-a", type=str, dest="arch", help="build arch") - parser_checkinstall.add_argument("--install-root", type=str, dest="install_root", help="check install root dir") + parser_checkinstall.add_argument("--install-root", type=str, dest="install_root", + help="check install root dir") parser_checkinstall.set_defaults(func=checkinstall) args = parser.parse_args() _ = not os.path.exists("log") and os.mkdir("log") - logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) + logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), + "../../conf/logger.conf")) logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") from src.utils.shell_cmd import shell_cmd_live diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 1b7840a..30568fd 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -21,9 +21,9 @@ import sys import logging.config import logging import json -import yaml import argparse import warnings +import yaml from yaml.error import YAMLError @@ -199,9 +199,6 @@ class Comment(object): logger.debug("comment: %s", content) for item in content: ac_result = ACResult.get_instance(item.get("result")) - # comments.append(self.__class__.comment_html_table_tr( - # item.get("name"), ac_result.emoji, ac_result.hint, item.get("link", ""), - # "markdown" if "link" in item else "", hashtag=False)) comments.append(self.__class__.comment_html_table_tr( item.get("name"), ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) @@ -264,7 +261,8 @@ if "__main__" == __name__: sys.exit(0) _ = not os.path.exists("log") and os.mkdir("log") - logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../conf/logger.conf")) + logger_conf_path = os.path.realpath(os.path.join(os.path.realpath(__file__), + "../../conf/logger.conf")) logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") @@ -283,7 +281,8 @@ if "__main__" == __name__: gp.delete_tag_of_pr(args.pr, "ci_processing") jp = JenkinsProxy(args.jenkins_base_url, args.jenkins_user, args.jenkins_api_token) - url, build_time, reason = jp.get_job_build_info(os.environ.get("JOB_NAME"), int(os.environ.get("BUILD_ID"))) + url, build_time, reason = jp.get_job_build_info(os.environ.get("JOB_NAME"), + int(os.environ.get("BUILD_ID"))) dd.set_attr_ctime("comment.job.ctime", build_time) dd.set_attr("comment.job.link", url) dd.set_attr("comment.trigger.reason", reason) @@ -310,7 +309,6 @@ if "__main__" == __name__: comment.comment_at(args.committer, gp) dd.set_attr_etime("comment.job.etime") - #dd.set_attr("comment.job.result", "successful") # suppress python warning warnings.filterwarnings("ignore") @@ -320,5 +318,6 @@ if "__main__" == __name__: # upload to es kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) query = {"term": {"id": args.comment_id}} - script = {"lang": "painless", "source": "ctx._source.comment = params.comment", "params": dd.to_dict()} + script = {"lang": "painless", "source": "ctx._source.comment = params.comment", + "params": dd.to_dict()} kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) -- Gitee From ad3b54be574f61ed7f0de4582267a29a4d0b4ff7 Mon Sep 17 00:00:00 2001 From: wwx1101975 Date: Wed, 24 Nov 2021 16:54:01 +0800 Subject: [PATCH 42/42] =?UTF-8?q?=E5=A2=9E=E5=8A=A0compare=20package?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E5=90=8E=E6=8E=A8=E9=80=81=E6=A8=A1=E6=9D=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/build/gitee_comment.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 30568fd..445c047 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -42,6 +42,7 @@ class Comment(object): self._up_builds = [] self._up_up_builds = [] self._get_upstream_builds(jenkins_proxy) + self._compare_package_result = True def comment_build(self, gitee_proxy): """ @@ -55,6 +56,18 @@ class Comment(object): return "\n".join(comments) + def comment_explain_template(self, gitee_proxy): + """ + 构建结果 + :param jenkins_proxy: + :param gitee_proxy: + :return: + """ + comments = self._comment_of_explain_template() + gitee_proxy.comment_pr(self._pr, "\n".join(comments)) + + return "\n".join(comments) + def comment_at(self, committer, gitee_proxy): """ 通知committer @@ -103,6 +116,7 @@ class Comment(object): comments.extend(self._comment_of_ac(self._up_up_builds[0])) if self._up_builds: comments.extend(self._comment_of_build(self._up_builds)) + #check_abi and compare_package comment comments.extend(self._comment_of_check_abi(self._up_builds)) comments.append("") @@ -159,6 +173,20 @@ class Comment(object): name, ac_result.emoji, ac_result.hint, "{}{}".format(build_url, "console"), build.buildno)) logger.info("build comment: %s", comments) + return comments + + def _comment_of_explain_template(self): + """ + compare package explain template + :param: + :return: + """ + comments = [] + comments.append('二进制变更检查失败原因说明模板') + comments.append('差异原因：') + comments.append('影响分析：') + + logger.info("compare package comment: %s", comments) return comments @@ -178,7 +206,7 @@ class Comment(object): return False for check_abi_comment_file in self._check_abi_comment_files: - logger.debug("check abi comment file: %s", check_abi_comment_file) + logger.debug("comment file: %s", check_abi_comment_file) if not os.path.exists(check_abi_comment_file): # check abi评论文件存在 continue for build in builds: @@ -202,9 +230,12 @@ class Comment(object): comments.append(self.__class__.comment_html_table_tr( item.get("name"), ac_result.emoji, ac_result.hint, "{}{}".format(build.get_build_url(), "console"), build.buildno)) + # 二进制变更检查比较成功 + if "compare_package" in item.get("name") and ac_result.emoji != SUCCESS: + self._compare_package_result = False break - logger.info("check abi comment: %s", comments) + logger.info("comment: %s", comments) return comments @@ -306,6 +337,8 @@ if "__main__" == __name__: dd.set_attr("comment.build.result", "failed") logger.info("comment: at committer......") + if not comment._compare_package_result: + comment.comment_explain_template(gp) comment.comment_at(args.committer, gp) dd.set_attr_etime("comment.job.etime") -- Gitee