From 29c88ef63074fc8124c47b8eab8d633039f1ff5f Mon Sep 17 00:00:00 2001 From: Jiachen Fan Date: Mon, 24 May 2021 14:03:36 +0800 Subject: [PATCH 01/39] add new free license perl remove gevent version to fix build error --- src/ac/acl/package_license/check_license.py | 2 +- src/ac/acl/package_license/config/Licenses.yaml | 6 ++++-- src/requirements | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/ac/acl/package_license/check_license.py b/src/ac/acl/package_license/check_license.py index fad13ee..28eeaf2 100644 --- a/src/ac/acl/package_license/check_license.py +++ b/src/ac/acl/package_license/check_license.py @@ -94,7 +94,7 @@ class CheckLicense(BaseCheck): else: logger.error("licenses in src:{} and in spec:{} are not same".format(self._license_in_src, self._license_in_spec)) - return FAILED + return WARNING def __call__(self, *args, **kwargs): """ diff --git a/src/ac/acl/package_license/config/Licenses.yaml b/src/ac/acl/package_license/config/Licenses.yaml index f0ba789..56f9723 100644 --- a/src/ac/acl/package_license/config/Licenses.yaml +++ b/src/ac/acl/package_license/config/Licenses.yaml @@ -638,8 +638,6 @@ Software Licenses: identifier: Artistic-1.0-cl8 - alias: - Artistic License 1.0 (Perl) - - (perl) - - "license: perl" identifier: Artistic-1.0-Perl - alias: - Lawrence Berkeley National Labs BSD variant license @@ -779,6 +777,10 @@ Software Licenses: - AFLv2.1 identifier: AFL-2.1 Free Licenses: + - alias: + - (perl) + - "license: perl" + identifier: Perl License - alias: - BSD Zero Clause License - Zero-Clause BSD diff --git a/src/requirements b/src/requirements index bb5b47e..7bcb633 100644 --- a/src/requirements +++ b/src/requirements @@ -3,7 +3,7 @@ jenkinsapi colorlog threadpool PyYAML -gevent==1.2.2 +gevent jsonpath mock tldextract -- Gitee From 4058aec6c6b6a02d1f14cada28e0c3fcb54970c4 Mon Sep 17 00:00:00 2001 From: Jiachen Fan Date: Mon, 31 May 2021 04:58:14 +0800 Subject: [PATCH 02/39] fix encode error add TCL license fix error from py2 to py3 add sleep to close connection --- .../acl/package_license/config/Licenses.yaml | 1 + src/ac/acl/package_license/package_license.py | 4 +-- src/ac/acl/package_yaml/check_repo.py | 4 +++ test/ac/acl/license/test_check_license.py | 17 ++++++------ test/ac/acl/package_yaml/test_check_repo.py | 27 +++++++++++-------- test/ac/acl/package_yaml/test_check_yaml.py | 8 +++--- 6 files changed, 36 insertions(+), 25 deletions(-) diff --git a/src/ac/acl/package_license/config/Licenses.yaml b/src/ac/acl/package_license/config/Licenses.yaml index 56f9723..96f3fbf 100644 --- a/src/ac/acl/package_license/config/Licenses.yaml +++ b/src/ac/acl/package_license/config/Licenses.yaml @@ -637,6 +637,7 @@ Software Licenses: - Artistic License 1.0 w/clause 8 identifier: Artistic-1.0-cl8 - alias: + - The "Artistic License" - Artistic License 1.0 (Perl) identifier: Artistic-1.0-Perl - alias: diff --git a/src/ac/acl/package_license/package_license.py b/src/ac/acl/package_license/package_license.py index 1735b4b..f64f7d5 100644 --- a/src/ac/acl/package_license/package_license.py +++ b/src/ac/acl/package_license/package_license.py @@ -157,11 +157,11 @@ class PkgLicense(object): for filename in filenames: if (filename.lower() in self.LICENSE_FILE_TARGET or self.LICENSE_TARGET_PAT.search(filename.lower())): - logger.info("scan the license target file: %s", filename) + logger.info("scan the license target file: %s", os.path.join(root, filename).replace(srcdir, "")) licenses_in_file.update( self.scan_licenses( os.path.join(root, filename))) - logger.info("all licenses from src: %s", ", ".join([data.encode("utf-8") for data in licenses_in_file])) + logger.info("all licenses from src: %s", ", ".join([data for data in licenses_in_file])) return licenses_in_file def scan_licenses(self, copying): diff --git a/src/ac/acl/package_yaml/check_repo.py b/src/ac/acl/package_yaml/check_repo.py index d7be84e..839fd0b 100644 --- a/src/ac/acl/package_yaml/check_repo.py +++ b/src/ac/acl/package_yaml/check_repo.py @@ -225,6 +225,8 @@ class MetacpanReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): logging.warning("illegal url: \"\"") return [] response = self.get_request_response(url) + if not response: + return [] resp_lines = response.text.splitlines() release_tags = [] tag_condition = "value=\"/release" @@ -328,6 +330,8 @@ class GnuftpReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): response = self.get_request_response(url) pattern = re.compile("href=\"(.*)\">(.*)") release_tags = [] + if not response: + return [] for line in response.text.splitlines(): search_result = pattern.search(line) if search_result: diff --git a/test/ac/acl/license/test_check_license.py b/test/ac/acl/license/test_check_license.py index 3ca7b26..ffab33c 100644 --- a/test/ac/acl/license/test_check_license.py +++ b/test/ac/acl/license/test_check_license.py @@ -17,7 +17,7 @@ """ import unittest -import mock +from unittest import mock import sys import os import types @@ -52,9 +52,9 @@ class TestCheckPkgLicense(unittest.TestCase): self._pkg_license.load_config() def decompress(self): self._gr.decompress_all() - check.get_work_tar_dir = types.MethodType(get_work_tar_dir, check, CheckLicense) - check.load_license_config = types.MethodType(load_license_config, check, CheckLicense) - check.decompress = types.MethodType(decompress, check, CheckLicense) + check.get_work_tar_dir = types.MethodType(get_work_tar_dir, check) + check.load_license_config = types.MethodType(load_license_config, check) + check.decompress = types.MethodType(decompress, check) def _test_check_license_in_spec(self, dir_key, predict): os.chdir(os.path.join(self.DIR_PATH, @@ -64,9 +64,9 @@ class TestCheckPkgLicense(unittest.TestCase): self.bind_func(cl) cl.load_license_config() self.assertEqual(cl.check_license_in_spec(), predict) - + def test_check_license_in_spec_none(self): - self._test_check_license_in_spec("no_spec", WARNING) + self._test_check_license_in_spec("no_spec", FAILED) def test_check_license_in_spec_succeed(self): self._test_check_license_in_spec("spec_success", SUCCESS) @@ -89,8 +89,9 @@ class TestCheckPkgLicense(unittest.TestCase): shutil.rmtree(cl.get_work_tar_dir()) def test_check_license_none(self): - self._test_check_license_in_src("no_src", WARNING) - + # 源码中不存在license由日志保存，不返回失败结果 + self._test_check_license_in_src("no_src", SUCCESS) + def test_check_license_in_src_succeed(self): self._test_check_license_in_src("src_success", SUCCESS) diff --git a/test/ac/acl/package_yaml/test_check_repo.py b/test/ac/acl/package_yaml/test_check_repo.py index 3de78eb..846d071 100644 --- a/test/ac/acl/package_yaml/test_check_repo.py +++ b/test/ac/acl/package_yaml/test_check_repo.py @@ -17,12 +17,13 @@ """ import unittest -import mock +from unittest import mock import os import yaml import logging.config import logging import shutil +from time import sleep from src.ac.acl.package_yaml.check_repo import ReleaseTagsFactory @@ -55,6 +56,7 @@ class TestGetReleaseTags(unittest.TestCase): return result def _get_test_tags(self, version): + sleep(2) yaml_content = self._load_yaml(self.TEST_YAML_DIR[version]) vc = yaml_content.get("version_control", "") sr = yaml_content.get("src_repo", "") @@ -65,9 +67,10 @@ class TestGetReleaseTags(unittest.TestCase): release_tags = self._get_test_tags("hg") self.assertEqual(len(release_tags) > 0, True) - def test_get_github_release_tags(self): - release_tags = self._get_test_tags("github") - self.assertEqual(len(release_tags) > 0, True) + # 当前测试用例中网址无法访问，待后续更新，暂时关闭该单测 + # def test_get_github_release_tags(self): + # release_tags = self._get_test_tags("github") + # self.assertEqual(len(release_tags) > 0, True) def test_get_git_release_tags(self): release_tags = self._get_test_tags("git") @@ -76,14 +79,15 @@ class TestGetReleaseTags(unittest.TestCase): def test_get_gitlab_gnome_release_tags(self): release_tags = self._get_test_tags("gitlab.gnome") self.assertEqual(len(release_tags) > 0, True) - + def test_get_svn_release_tags(self): release_tags = self._get_test_tags("svn") self.assertEqual(len(release_tags) > 0, True) - def test_get_metacpan_release_tags(self): - release_tags = self._get_test_tags("metacpan") - self.assertEqual(len(release_tags) > 0, True) + # 当前测试用例中网址无法访问，待后续更新，暂时关闭该单测 + # def test_get_metacpan_release_tags(self): + # release_tags = self._get_test_tags("metacpan") + # self.assertEqual(len(release_tags) > 0, True) def test_get_pypi_release_tags(self): release_tags = self._get_test_tags("pypi") @@ -97,9 +101,10 @@ class TestGetReleaseTags(unittest.TestCase): release_tags = self._get_test_tags("gitee") self.assertEqual(len(release_tags) > 0, True) - def test_get_gnu_ftp_release_tags(self): - release_tags = self._get_test_tags("gnu-ftp") - self.assertEqual(len(release_tags) > 0, True) + # 当前测试用例中网址无法访问，待后续更新，暂时关闭该单测 + # def test_get_gnu_ftp_release_tags(self): + # release_tags = self._get_test_tags("gnu-ftp") + # self.assertEqual(len(release_tags) > 0, True) if __name__ == '__main__': work_dir = os.getcwd() diff --git a/test/ac/acl/package_yaml/test_check_yaml.py b/test/ac/acl/package_yaml/test_check_yaml.py index 0a189ba..57f04fa 100644 --- a/test/ac/acl/package_yaml/test_check_yaml.py +++ b/test/ac/acl/package_yaml/test_check_yaml.py @@ -17,7 +17,7 @@ """ import unittest -import mock +from unittest import mock import sys import os import types @@ -48,7 +48,7 @@ class TestCheckYamlField(unittest.TestCase): self.cy = CheckPackageYaml("", "", "") def set_yaml(self, file): self._gr.yaml_file = file - self.cy.set_yaml = types.MethodType(set_yaml, self.cy, CheckPackageYaml) # python3该接口有变化 为实例动态绑定接口 + self.cy.set_yaml = types.MethodType(set_yaml, self.cy) def test_none_file(self): self.cy.set_yaml(None) @@ -104,8 +104,8 @@ class TestCheckYamlRepo(unittest.TestCase): self._gr.yaml_file = file def set_spec(self, file): self._spec = RPMSpecAdapter(file) - self.cy.set_yaml = types.MethodType(set_yaml, self.cy, CheckPackageYaml) # python3该接口有变化 为实例动态绑定接口 - self.cy.set_spec = types.MethodType(set_spec, self.cy, CheckPackageYaml) # python3该接口有变化 为实例动态绑定接口 + self.cy.set_yaml = types.MethodType(set_yaml, self.cy) + self.cy.set_spec = types.MethodType(set_spec, self.cy) def test_none_file(self): self.cy.set_yaml(None) -- Gitee From 3f3128aa77d13f55a2e58ccd8c09ce36c0bdfb30 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Mon, 19 Apr 2021 11:03:26 +0800 Subject: [PATCH 03/39] use python3 stop report to es make address clickable, because jenkins console add special char append each line use local openjdk add libabigail and perl-xml-structured packages to ci common image delete useless code docstring kafka producer need serialize string key for python3 fix bug, send data to kafka miss param key add dockerfile for bishengjdk riscv port package at exclude state when init --- src/ac/acl/package_yaml/check_repo.py | 4 +- src/ac/acl/sca/check_sca.py | 2 +- src/ac/acl/spec/check_spec.py | 2 +- src/ac/common/gitee_repo.py | 2 +- src/ac/framework/ac.py | 13 +-- src/build/gitee_comment.py | 2 - src/build/osc_build_k8s.py | 7 +- src/dockerfile/README.md | 31 +++++ src/dockerfile/{obs-base => ci-base-ubuntu} | 0 src/dockerfile/ci-bisheng-riscv | 106 ++++++++++++++++++ .../{jenkins-openeuler => ci-common} | 11 +- src/dockerfile/ci-soe-base | 47 ++++++++ src/dockerfile/{obs => ci-ubuntu} | 0 src/dockerfile/openjdk-openeuler | 8 +- src/jobs/jenkins_job.py | 4 +- src/proxy/git_proxy.py | 19 +++- src/proxy/jenkins_patch.py | 4 +- src/proxy/kafka_proxy.py | 1 + src/proxy/obs_proxy.py | 7 +- src/requirements | 3 +- src/utils/shell_cmd.py | 1 + 21 files changed, 237 insertions(+), 37 deletions(-) create mode 100644 src/dockerfile/README.md rename src/dockerfile/{obs-base => ci-base-ubuntu} (100%) create mode 100644 src/dockerfile/ci-bisheng-riscv rename src/dockerfile/{jenkins-openeuler => ci-common} (74%) create mode 100644 src/dockerfile/ci-soe-base rename src/dockerfile/{obs => ci-ubuntu} (100%) diff --git a/src/ac/acl/package_yaml/check_repo.py b/src/ac/acl/package_yaml/check_repo.py index 9541ea3..d7be84e 100644 --- a/src/ac/acl/package_yaml/check_repo.py +++ b/src/ac/acl/package_yaml/check_repo.py @@ -22,7 +22,7 @@ import logging import re -import urlparse +import urllib.parse as urlparse import requests import json import subprocess @@ -537,4 +537,4 @@ class ReleaseTagsFactory(object): return: class """ release_tags = ReleaseTagsFactory.VERSION_CTRL_GETTER_MAPPING.get(version_control, DefaultReleaseTags) - return release_tags(version_control) \ No newline at end of file + return release_tags(version_control) diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index d4d9af9..da04991 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -82,7 +82,7 @@ class CheckSCA(BaseCheck): # 保存详细结果到web server if not result: self.save_scanoss_result(scan.html) - logger.warning("click to view scanoss detail: {}".format(self._scanoss_result_repo_path)) + logger.warning("click {} view scanoss detail".format(self._scanoss_result_repo_path)) return SUCCESS if result else FAILED diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index 6502e91..289969b 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -123,7 +123,7 @@ class CheckSpec(BaseCheck): if not homepage: return SUCCESS - for _ in xrange(retrying): + for _ in range(retrying): if 0 == do_requests("get", homepage, timeout=timeout): return SUCCESS time.sleep(interval) diff --git a/src/ac/common/gitee_repo.py b/src/ac/common/gitee_repo.py index 3432d4f..0f4527f 100644 --- a/src/ac/common/gitee_repo.py +++ b/src/ac/common/gitee_repo.py @@ -139,7 +139,7 @@ class GiteeRepo(object): if os.path.isdir(os.path.join(self._decompress_dir, filename))] + ["."]: if patch_dir.startswith(".git"): continue - for leading in xrange(max_leading + 1): + for leading in range(max_leading + 1): logger.debug("try dir {} -p{}".format(patch_dir, leading)) if GitProxy.apply_patch_at_dir(os.path.join(self._decompress_dir, patch_dir), os.path.join(self._work_dir, patch), leading): diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index 9beda0b..da90427 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -136,7 +136,7 @@ class AC(object): :return: """ for filename in os.listdir(acl_dir): - if os.path.isdir(os.path.join(acl_dir, filename)): + if filename != "__pycache__" and os.path.isdir(os.path.join(acl_dir, filename)): self._ac_check_elements[filename] = {} # don't worry, using default when checking def load_check_elements_from_conf(self, conf_file, community): @@ -223,7 +223,7 @@ if "__main__" == __name__: logger = logging.getLogger("ac") logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git".format(args.community, args.repo)) + logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.community, args.repo)) logger.info("clone depth 4") logger.info("checking out pull request {}".format(args.pr)) @@ -254,7 +254,6 @@ if "__main__" == __name__: logging.getLogger("elasticsearch").setLevel(logging.WARNING) logging.getLogger("kafka").setLevel(logging.WARNING) - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) # download repo @@ -266,9 +265,7 @@ if "__main__" == __name__: dd.set_attr_etime("access_control.scm.etime") dd.set_attr_etime("access_control.job.etime") - #dd.set_attr("access_control.job.result", "successful") - ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) - kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) + kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) logger.info("fetch finished -") sys.exit(-1) else: @@ -299,6 +296,4 @@ if "__main__" == __name__: ac.save(args.output) dd.set_attr_etime("access_control.job.etime") - #dd.set_attr("access_control.job.result", "successful") - ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) - kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) + kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 56ef4c0..442ebbb 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -315,9 +315,7 @@ if "__main__" == __name__: logging.getLogger("kafka").setLevel(logging.WARNING) # upload to es - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.comment = params.comment", "params": dd.to_dict()} - #ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 98cccaa..eede4c6 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -66,7 +66,7 @@ class SinglePackageBuild(object): 需要构建obs repo列表 :return: list """ - return OBSProxy.list_repos_of_arch(project, self._package, self._arch) + return OBSProxy.list_repos_of_arch(project, self._package, self._arch, show_exclude=True) def build_obs_repos(self, project, repos, work_dir, code_dir): """ @@ -251,7 +251,7 @@ if "__main__" == __name__: logger = logging.getLogger("build") logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git".format(args.owner, args.repo)) + logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.owner, args.repo)) logger.info("clone depth 1") logger.info("checking out pull request {}".format(args.pr)) @@ -272,7 +272,6 @@ if "__main__" == __name__: logging.getLogger("elasticsearch").setLevel(logging.WARNING) logging.getLogger("kafka").setLevel(logging.WARNING) - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) # download repo @@ -291,7 +290,6 @@ if "__main__" == __name__: query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) sys.exit(-1) else: @@ -311,6 +309,5 @@ if "__main__" == __name__: # upload to es query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) sys.exit(rs) diff --git a/src/dockerfile/README.md b/src/dockerfile/README.md new file mode 100644 index 0000000..b4af737 --- /dev/null +++ b/src/dockerfile/README.md @@ -0,0 +1,31 @@ +# docker images +## openeuler +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.03-lts + +## openjdk 11-jdk-stretch based on openeuler +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.03-lts + +> Dockerfile: openjdk-openeuler + +## openeuler ci image based on openjdk +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.03-lts + +> Dockerfile: ci-common + +## src-openeuler ci image based on openjdk +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/soe:base + +> Dockerfile: ci-soe-base + +> choose an openjdk image that from stable openeuler version + +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/soe:{version} diff --git a/src/dockerfile/obs-base b/src/dockerfile/ci-base-ubuntu similarity index 100% rename from src/dockerfile/obs-base rename to src/dockerfile/ci-base-ubuntu diff --git a/src/dockerfile/ci-bisheng-riscv b/src/dockerfile/ci-bisheng-riscv new file mode 100644 index 0000000..b4169bb --- /dev/null +++ b/src/dockerfile/ci-bisheng-riscv @@ -0,0 +1,106 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +# Install qemu 5.2.0 +RUN set -eux; \ + yum update; yum clean all; \ + yum -y groupinstall "Development Tools"; \ + yum -y install pixman-devel zlib-* glib2 glib2-devel perl tar xz zip unzip wget; \ + cd /usr/src; \ + wget -q http://121.36.53.23/bisheng_riscv/ninja-linux.zip; \ + unzip ninja-linux.zip; \ + cp ninja /usr/bin; \ + wget -q http://121.36.53.23/bisheng_riscv/qemu-5.2.0.tar.xz; \ + tar xJf qemu-5.2.0.tar.xz; \ + cd qemu-5.2.0 && mkdir build && cd build; \ + ../configure --target-list=riscv64-linux-user,riscv64-softmmu; \ + make install -j; \ + rm -rf /usr/src/qemu-5.2.0* /usr/src/ninja*; + +# RISCV toolchain +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/riscv_toolchain_linux64.tar.xz; \ + tar xJf riscv_toolchain_linux64.tar.xz; \ + rm riscv_toolchain_linux64.tar.xz; + +# fedora sysroot +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/fedora28_riscv_smlroot.tar.xz; \ + tar xJf fedora28_riscv_smlroot.tar.xz; \ + ln -s /opt/fedora28_riscv_root/usr/lib64/ld-linux-riscv64-lp64d.so.1 /lib/ld-linux-riscv64-lp64d.so.1; \ + rm fedora28_riscv_smlroot.tar.xz; + +# Packages for openjdk build +RUN set -eux; \ + yum -y install java-1.8.0-openjdk-devel cups-devel libXtst-devel libXt-devel libXrender-devel libXi-devel alsa-lib-devel libffi-devel elfutils-libelf-devel libstdc++-static libatomic + +# Test tool packages +RUN set -eux; \ + yum -y install gcc gcc-c++ unzip zlib-devel perl-devel libcurl-devel openssl-devel mercurial perl perl-Test-Simple xorg-x11-server-Xorg xorg-x11-server-Xvfb + +# JTREG +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/jtreg-4.2.0-tip.tar.gz; \ + mkdir jtreg-4; tar -xf jtreg-4.2.0-tip.tar.gz -C jtreg-4 --strip-components 1; \ + wget -q http://121.36.53.23/bisheng_riscv/jtreg-5.1-b01.tar.gz; \ + mkdir jtreg-5; tar -xf jtreg-5.1-b01.tar.gz -C jtreg-5 --strip-components 1; \ + rm jtreg*.tar.gz; + +# AdoptOpenJDK 8/10/11/16 +RUN mkdir -p /usr/lib/jvm && cd /usr/lib/jvm; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK8U-jdk_x64_linux_hotspot_8u292b10.tar.gz -O jdk-8.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK10U-jdk_x64_linux_hotspot_10.0.2_13.tar.gz -O jdk-10.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK11U-jdk_x64_linux_hotspot_11.0.11_9.tar.gz -O jdk-11.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK16U-jdk_x64_linux_hotspot_16.0.1_9.tar.gz -O jdk-16.tar.gz; \ + for tar in *.tar.gz; do tar xf $tar; done; \ + rm *.tar.gz; \ + ln -s jdk8* jdk-8; \ + ln -s jdk-10* jdk-10; \ + ln -s jdk-11* jdk-11; \ + ln -s jdk-16* jdk-16; + +ENV \ + JDK8_BOOT_DIR="/usr/lib/jvm/jdk-8" \ + JDK10_BOOT_DIR="/usr/lib/jvm/jdk-10" \ + JDK11_BOOT_DIR="/usr/lib/jvm/jdk-11" \ + JDK16_BOOT_DIR="/usr/lib/jvm/jdk-16" \ + JTREG4_DIR="/opt/jtreg-4" \ + JTREG5_DIR="/opt/jtreg-5" + +# Jenkins Agent +RUN set -eux; \ + yum install -y python3 python3-pip cpio bsdtar expect openssh sudo shadow curl wget vim git strace + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] diff --git a/src/dockerfile/jenkins-openeuler b/src/dockerfile/ci-common similarity index 74% rename from src/dockerfile/jenkins-openeuler rename to src/dockerfile/ci-common index 98ffa41..b99d662 100644 --- a/src/dockerfile/jenkins-openeuler +++ b/src/dockerfile/ci-common @@ -1,5 +1,8 @@ # replace VERSION before run -FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler-VERSION/openjdk:11-jdk-stretch +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +RUN set -eux; \ + yum install -y python3 python3-pip cpio bsdtar expect openssh sudo shadow curl wget vim git strace ARG VERSION=4.3 ARG user=jenkins @@ -13,7 +16,9 @@ RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins- && chmod 644 /usr/share/jenkins/agent.jar \ && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar -COPY jenkins-agent /usr/local/bin/jenkins-agent +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + RUN chmod a+rx /usr/local/openjdk-11 \ && chmod a+rx /usr/local/bin/jenkins-agent \ && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave @@ -33,5 +38,3 @@ WORKDIR ${AGENT_WORKDIR} ENTRYPOINT ["jenkins-agent"] - -# swr.cn-north-4.myhuaweicloud.com/openeuler/jenkins/openeuler:base diff --git a/src/dockerfile/ci-soe-base b/src/dockerfile/ci-soe-base new file mode 100644 index 0000000..b6332ad --- /dev/null +++ b/src/dockerfile/ci-soe-base @@ -0,0 +1,47 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +# golint not ready yet +RUN set -eux; \ + yum install -y curl sudo cpio bsdtar expect openssh shadow vim git strace python3 python3-pip build osc pylint + +# libabigail perl-XML-Structured not ready yet, use SP2 dailybuild +RUN set -eux; \ + arch="$(arch)"; \ + yum install -y http://121.36.53.23/SP2/libabigail-1.6-4.oe1.$arch.rpm; \ + yum install -y http://121.36.53.23/SP2/perl-XML-Structured-1.3-2.oe1.$arch.rpm + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] diff --git a/src/dockerfile/obs b/src/dockerfile/ci-ubuntu similarity index 100% rename from src/dockerfile/obs rename to src/dockerfile/ci-ubuntu diff --git a/src/dockerfile/openjdk-openeuler b/src/dockerfile/openjdk-openeuler index aaf5914..7ca38a9 100644 --- a/src/dockerfile/openjdk-openeuler +++ b/src/dockerfile/openjdk-openeuler @@ -2,7 +2,7 @@ FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:VERSION RUN set -eux; \ - yum install -y tar wget python3 expect osc vim openssh shadow git + yum install -y tar wget # Default to UTF-8 file.encoding ENV LANG C.UTF-8 @@ -33,9 +33,11 @@ RUN set -eux; \ # this "case" statement is generated via "update.sh" case "$arch" in \ # arm64v8 - arm64 | aarch64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_aarch64_linux_11.0.8_10.tar.gz ;; \ + arm64 | aarch64) downloadUrl=http://121.36.53.23/AdoptOpenJDK/openjdk_aarch64.tgz ;; \ # amd64 - amd64 | i386:x86-64 | x86_64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_x64_linux_11.0.8_10.tar.gz ;; \ + amd64 | i386:x86-64 | x86_64) downloadUrl=http://121.36.53.23/AdoptOpenJDK/openjdk_x64.tgz ;; \ + #arm64 | aarch64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_aarch64_linux_11.0.8_10.tar.gz ;; \ + #amd64 | i386:x86-64 | x86_64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_x64_linux_11.0.8_10.tar.gz ;; \ # fallback *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ esac; \ diff --git a/src/jobs/jenkins_job.py b/src/jobs/jenkins_job.py index 7ecab8a..3978493 100755 --- a/src/jobs/jenkins_job.py +++ b/src/jobs/jenkins_job.py @@ -68,7 +68,7 @@ class JenkinsJobs(object): """ batch = (len(target_jobs) + concurrency - 1) / concurrency _failed_jobs = [] - for index in xrange(batch): + for index in range(batch): works = [gevent.spawn(self.dispatch, action, job, jenkins_proxy) for job in target_jobs[index * concurrency: (index + 1) * concurrency]] logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) @@ -86,7 +86,7 @@ class JenkinsJobs(object): failed_jobs = run_once(real_jobs) - for index in xrange(retry): + for index in range(retry): if not failed_jobs: break logger.info("{} jobs failed, retrying {}/{}".format(len(failed_jobs), index + 1, retry)) diff --git a/src/proxy/git_proxy.py b/src/proxy/git_proxy.py index e2c644b..87fe232 100644 --- a/src/proxy/git_proxy.py +++ b/src/proxy/git_proxy.py @@ -1,7 +1,24 @@ # -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-09-23 +# Description: git api proxy +# ********************************************************************************** +""" + import os import logging -from cStringIO import StringIO +from io import StringIO import retrying from src.utils.shell_cmd import shell_cmd_live diff --git a/src/proxy/jenkins_patch.py b/src/proxy/jenkins_patch.py index 4fa3ef2..1b73d44 100644 --- a/src/proxy/jenkins_patch.py +++ b/src/proxy/jenkins_patch.py @@ -1,5 +1,5 @@ # -*- encoding=utf-8 -*- -from urllib import quote as urlquote +from urllib.parse import quote from jenkinsapi.jenkinsbase import JenkinsBase @@ -11,7 +11,7 @@ def resolve_job_folders(self, jobs): jobs.remove(job) jobs += self.process_job_folder(job, self.baseurl) else: - job["url"] = '%s/job/%s' % (self.baseurl, urlquote(job['name'])) + job["url"] = '%s/job/%s' % (self.baseurl, quote(job['name'])) return jobs diff --git a/src/proxy/kafka_proxy.py b/src/proxy/kafka_proxy.py index 9ff7ff7..c7dcde4 100644 --- a/src/proxy/kafka_proxy.py +++ b/src/proxy/kafka_proxy.py @@ -37,6 +37,7 @@ class KafkaProducerProxy(object): self._timeout = timeout self._kp = kafka.KafkaProducer(bootstrap_servers=brokers, + key_serializer=str.encode, value_serializer=lambda v:json.dumps(v).encode("utf-8")) def send(self, topic, key=None, value=None): diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index eee8ad1..935d944 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -51,14 +51,15 @@ class OBSProxy(object): return rs @staticmethod - def list_repos_of_arch(project, package, arch): + def list_repos_of_arch(project, package, arch, show_exclude=False): """ 获取包的repo列表 :param project: :param package: :return: """ - cmd = "osc results {} {} -a {}".format(project, package, arch) + cmd = "osc results {} {} {} -a {}".format( + "--show-exclude" if show_exclude else "", project, package, arch) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: logger.debug("list obs repos of arch error, {}".format(ret)) @@ -120,7 +121,7 @@ class OBSProxy(object): :return: """ package_path = "{}/{}".format(project, package) - cmd = "cd {}; osc build {} {} {} --no-verify --clean".format( + cmd = "cd {}; osc build {} {} {} --no-verify --clean --noservice".format( package_path, repo, arch, "--disable-debuginfo" if not debug else "") logger.info("osc build {} {} {} --no-verify --clean".format( diff --git a/src/requirements b/src/requirements index 7bcb633..d3fd573 100644 --- a/src/requirements +++ b/src/requirements @@ -8,6 +8,7 @@ jsonpath mock tldextract chardet -kafka +kafka-python elasticsearch retrying +scanoss-scanner diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index d3bc10c..dec8843 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -42,6 +42,7 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False while True: line = p.stdout.readline() if line: + line = line.decode("utf-8") line = line.strip() no_fmt_logger.info(line) if verbose else no_fmt_logger.debug(line) if cap_out and line and line != "\n": -- Gitee From b8fca376cde391350a876fdcee86ef2dbad49213 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Mon, 31 May 2021 18:18:03 +0800 Subject: [PATCH 04/39] support feature: check install if no repo build, regard as fail --- src/build/build_rpm_package.py | 21 +++++++++ src/build/extra_work.py | 55 +++++++++++++++++++++- src/build/obs_repo_source.py | 86 ++++++++++++++++++++++++++++++++++ src/build/osc_build_k8s.py | 7 +++ 4 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 src/build/obs_repo_source.py diff --git a/src/build/build_rpm_package.py b/src/build/build_rpm_package.py index e33d2e1..3f2d845 100755 --- a/src/build/build_rpm_package.py +++ b/src/build/build_rpm_package.py @@ -180,3 +180,24 @@ class BuildRPMPackage(object): for filename in filenames: name = self.extract_rpm_name(filename) self._rpm_packages["srpm"][name] = {"name": name, "fullname": filename} + + def iter_all_rpm(self): + """ + 遍历所有rpm包，返回包在local的路径 + :return: + """ + packages = self._rpm_packages.get("rpm", {}) + for name in packages: + package = packages[name] + yield name, os.path.join(self._rpmbuild_dir, "RPMS", package["arch"], package["fullname"]) + + def iter_all_srpm(self): + """ + 遍历所有source rpm包，返回包在local的路径 + :return: + """ + packages = self._rpm_packages.get("rpm", {}) + + for name in packages: + package = packages[name] + yield name, os.path.join(self._rpmbuild_dir, "SRPMS", package["fullname"]) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 39f8e90..1e22f43 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -22,6 +22,8 @@ import logging.config import logging import yaml +from src.build.obs_repo_source import OBSRepoSource + class ExtraWork(object): """ @@ -156,10 +158,57 @@ class ExtraWork(object): except IOError: logger.exception("save check abi comment exception") + def check_install_rpm(self, branch_name, arch, install_root): + """ + 检查生成的rpm是否可以安装 + :param branch_name: 分支名 + :param arch: cpu架构 + :param install_root: 安装根路径 + :return: + """ + logger.info("*** start check install start ***") + + # 1. prepare install root directory + _ = not os.path.exists(install_root) and os.mkdir(install_root) + logger.info("create install root directory: {}".format(install_root)) + + repo_name_prefix = "check_install" + + # 2. prepare repo + repo_source = OBSRepoSource("http://119.3.219.20:82") # obs 实时构建repo地址 + repo_config = repo_source.generate_repo_info(branch_name, arch, "check_install") + logger.info("repo source config:\n{}".format(repo_config)) + + # write to /etc/yum.repos.d + with open("obs_realtime.repo", "w+") as f: + f.write(repo_config) + + # 3. dnf install using repo name start with "check_install" + names = [] + packages = [] + for name, package in self._rpm_package.iter_all_rpm(): + # ignore debuginfo rpm + if "debuginfo" in name or "debugsource" in name: + logger.debug("ignore debug rpm: {}".format(name)) + continue + names.append(name) + packages.append(package) + + logger.info("install rpms: {}".format(names)) + if packages: + check_install_cmd = "sudo dnf install -y --disablerepo=* --enablerepo={}* --installroot={}" \ + "--setopt=reposdir=. {}".format(repo_name_prefix, install_root, " ".join(packages)) + ret, _, _ = shell_cmd_live(check_install_cmd, verbose=True) + if ret: + logger.error("install rpms error, {}, {}".format(ret, err)) + else: + logger.info("install rpm success") + + if "__main__" == __name__: args = argparse.ArgumentParser() - args.add_argument("-f", type=str, dest="func", choices=("notify", "checkabi"), help="function") + args.add_argument("-f", type=str, dest="func", choices=("notify", "checkabi", "checkinstall"), help="function") args.add_argument("-p", type=str, dest="package", help="obs package") args.add_argument("-a", type=str, dest="arch", help="build arch") @@ -180,6 +229,8 @@ if "__main__" == __name__: args.add_argument("-b", type=str, dest="obs_repo_url", help="obs repo where rpm saved") args.add_argument("-s", type=str, dest="obs_addr", help="obs address") args.add_argument("-r", type=str, dest="branch_name", help="obs project name") + + args.add_argument("--install-root", type=str, dest="install_root", help="check install root dir") args = args.parse_args() _ = not os.path.exists("log") and os.mkdir("log") @@ -203,3 +254,5 @@ if "__main__" == __name__: # run before copy rpm to rpm repo ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, args.obs_addr, args.branch_name, args.obs_repo_url) + elif args.func == "checkinstall": + ew.check_install_rpm(args.branch_name, args.arch, args.install_root) diff --git a/src/build/obs_repo_source.py b/src/build/obs_repo_source.py new file mode 100644 index 0000000..2c1db12 --- /dev/null +++ b/src/build/obs_repo_source.py @@ -0,0 +1,86 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-05-27 +# Description: obs repo as dnf source +# ********************************************************************************** +""" +from src.proxy.requests_proxy import do_requests +import logging + +logger = logging.getLogger("common") + + +class OBSRepoSource(object): + """ + 生成obs实时仓作为rpm源的配置 + """ + def __init__(self, repo_host): + """ + + :param repo_host: obs仓库host + """ + self._current_repo_host = repo_host + + @staticmethod + def repo_format(repo_id, repo_name, repo_baseurl): + """ + repo内容格式 + :param repo_id: + :param repo_name: + :param repo_baseurl: + :return: + """ + return "[{}]\nname={}\nbaseurl={}\nenabled=1\ngpgcheck=0\n".format(repo_id, repo_name, repo_baseurl) + + def generate_repo_info(self, branch, arch, repo_name_prefix): + """ + 不同的分支生成不同的repo + :param branch: + :param arch: + :param repo_name_prefix: + :return: + """ + if branch == "master": + obs_path_part = "openEuler:/Mainline" + elif "openstack" in branch: + branch = branch.replace("_oe-", "_openEuler-") # openEuler abbr. + vendor, openstack, os = branch.split("_") + obs_path_part = (":/").join( + [os.replace("-", ":/"), vendor.replace("-", ":/"), openstack.replace("-", ":/")]) + else: + obs_path_part = branch.replace("-", ":/") + + logger.debug("branch={}, obs_path_part={}".format(branch, obs_path_part)) + + repo_config = "" + + # main + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part, arch) + if do_requests("GET", url) == 0: + logger.debug("add man repo: {}".format(url)) + repo_config += self.repo_format(repo_name_prefix + "_main", repo_name_prefix + "_main", url) + + # epol + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part + ":/Epol", arch) + if do_requests("GET", url) == 0: + logger.debug("add epol repo: {}".format(url)) + repo_config += self.repo_format(repo_name_prefix + "_epol", repo_name_prefix + "_epol", url) + + # extras + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part + ":/Extras", arch) + if do_requests("GET", url) == 0: + logger.debug("add extras repo: {}".format(url)) + repo_config += self.repo_format(repo_name_prefix + "_extras", repo_name_prefix + "_extras", url)\ + + return repo_config diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index eede4c6..d80b2ae 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -199,6 +199,7 @@ class SinglePackageBuild(object): logger.error("branch \"{}\" not support yet".format(self._branch)) return 1 + has_any_repo_build = False for project in self.GITEE_BRANCH_PROJECT_MAPPING.get(self._branch): logger.debug("start build project {}".format(project)) @@ -208,6 +209,7 @@ class SinglePackageBuild(object): continue logger.debug("build obs repos: {}".format(obs_repos)) + has_any_repo_build = True ret = self.build_obs_repos(project, obs_repos, work_dir, code_dir) if ret > 0: logger.debug("build run return {}".format(ret)) @@ -216,6 +218,11 @@ class SinglePackageBuild(object): else: logger.info("build {} {} {} ... {}".format(project, self._package, self._arch, "ok")) + # if no repo build, regard as fail + if not has_any_repo_build: + logger.error("package not in any obs projects, please add package into obs") + return 1 + return 0 -- Gitee From e2379bd287fa56214851ee7eb0c350cf9544b9d2 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Tue, 1 Jun 2021 11:38:09 +0800 Subject: [PATCH 05/39] update ac README --- src/ac/README.md | 30 +++++++++++++++++++++++++----- src/build/extra_work.py | 4 ++-- src/build/obs_repo_source.py | 2 +- 3 files changed, 28 insertions(+), 8 deletions(-) diff --git a/src/ac/README.md b/src/ac/README.md index cc37080..7a18f15 100644 --- a/src/ac/README.md +++ b/src/ac/README.md @@ -1,28 +1,39 @@ # 门禁检查 ## 如何加入检查项 -1. 在ci_check/src/ac目录下新建文件夹 -2. 在ac_conf.yaml中增加配置项，可选 +1. 在ci_check/src/ac目录下新建文件夹放置检查项代码 +2. 在ac_conf.yaml中增加配置项 ### 配置文件说明 ```yaml 示例=> spec: # ac项目名称 - hint: check_spec # gitee中显示名，缺省使用check_+项目名称 + hint: check_spec # gitee中显示检查项名称，缺省使用check_+项目名称 module: spec.check_spec # ac项目模块名称，缺省使用"项目名称+check_+项目名称" - entry: Entry # ac项目入口，入口属性具备callable，缺省使用"run" + entry: Entry # ac项目入口类名称，继承BaseCheck类，可自定义__callable__方法 exclude: true # 忽略该项检查 ignored: [] # ac项目内忽略的检查项，就算失败也不影响最终ac项目结果 + allow_list: [] # 只有出现在allow_list的包才执行当前检查项 + deny_list:[] # 出现在deny_list的包不执行当前检查项 ``` ### entry实现模板 ```yaml -class Entry(object): +from src.ac.framework.ac_base import BaseCheck +from src.ac.framework.ac_result import FAILED, SUCCESS, WARNING + + +class Entry(BaseCheck): def __call__(self, *args, **kwargs): # do the work ... + + def check_case_a(self): + # do the check + + return SUCCESS ``` ### 检查结果 @@ -32,3 +43,12 @@ class Entry(object): | 0 | SUCCESS | :white_check_mark:| | 1 | WARNING | :bug: | | 2 | FAILED | :x:| + +## 支持的检查项 +| 检查项 | 目录 | 描述 | +| --- | --- | --- | +| spec文件 | spec | 检查homepage是否可以访问、版本号单调递增、检查补丁文件是否存在| +| 代码风格 | code | 检查压缩包文件、检查补丁是否可以使用、执行linter工具 | +| yaml文件 | package_yaml | | +| license检查 | package_license | | +| 代码片段检查 | sca | 目前只针对自研项目 | \ No newline at end of file diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 1e22f43..3c99463 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -197,8 +197,8 @@ class ExtraWork(object): logger.info("install rpms: {}".format(names)) if packages: check_install_cmd = "sudo dnf install -y --disablerepo=* --enablerepo={}* --installroot={}" \ - "--setopt=reposdir=. {}".format(repo_name_prefix, install_root, " ".join(packages)) - ret, _, _ = shell_cmd_live(check_install_cmd, verbose=True) + " --setopt=reposdir=. {}".format(repo_name_prefix, install_root, " ".join(packages)) + ret, _, err = shell_cmd_live(check_install_cmd, verbose=True) if ret: logger.error("install rpms error, {}, {}".format(ret, err)) else: diff --git a/src/build/obs_repo_source.py b/src/build/obs_repo_source.py index 2c1db12..748f1da 100644 --- a/src/build/obs_repo_source.py +++ b/src/build/obs_repo_source.py @@ -68,7 +68,7 @@ class OBSRepoSource(object): # main url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part, arch) if do_requests("GET", url) == 0: - logger.debug("add man repo: {}".format(url)) + logger.debug("add main repo: {}".format(url)) repo_config += self.repo_format(repo_name_prefix + "_main", repo_name_prefix + "_main", url) # epol -- Gitee From 673156c3dc165fa58715ea802ab7b385bd4e1e29 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Tue, 1 Jun 2021 19:47:51 +0800 Subject: [PATCH 06/39] openstack need common and base repo generate buddy info ignore Bak dir --- src/build/obs_repo_source.py | 21 +++++++++++++++++++-- src/jobs/obs_meta_strategy.py | 2 +- src/utils/shell_cmd.py | 8 ++++++-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/build/obs_repo_source.py b/src/build/obs_repo_source.py index 748f1da..3548350 100644 --- a/src/build/obs_repo_source.py +++ b/src/build/obs_repo_source.py @@ -51,6 +51,8 @@ class OBSRepoSource(object): :param repo_name_prefix: :return: """ + repo_config = "" + if branch == "master": obs_path_part = "openEuler:/Mainline" elif "openstack" in branch: @@ -58,13 +60,28 @@ class OBSRepoSource(object): vendor, openstack, os = branch.split("_") obs_path_part = (":/").join( [os.replace("-", ":/"), vendor.replace("-", ":/"), openstack.replace("-", ":/")]) + obs_path_part_common = (":/").join( + [os.replace("-", ":/"), vendor.replace("-", ":/"), "openstack:/common"]) + obs_path_part_base = (":/").join([os.replace("-", ":/")]) + + # openstack need common and base + if "openstack-common" != openstack: + # openstack common + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part_common, arch) + if do_requests("GET", url) == 0: + logger.debug("add openstack common repo: {}".format(url)) + repo_config += self.repo_format("openstack_common", repo_name_prefix + "_openstack_common", url) + + # openstack base + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part_base, arch) + if do_requests("GET", url) == 0: + logger.debug("add openstack base repo: {}".format(url)) + repo_config += self.repo_format("openstack_base", repo_name_prefix + "_openstack_base", url) else: obs_path_part = branch.replace("-", ":/") logger.debug("branch={}, obs_path_part={}".format(branch, obs_path_part)) - repo_config = "" - # main url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part, arch) if do_requests("GET", url) == 0: diff --git a/src/jobs/obs_meta_strategy.py b/src/jobs/obs_meta_strategy.py index 2b38426..6b30620 100755 --- a/src/jobs/obs_meta_strategy.py +++ b/src/jobs/obs_meta_strategy.py @@ -72,7 +72,7 @@ class ObsMetaStrategy(object): index = 0 for dirpath, dirnames, filenames in os.walk(self._obs_meta_path): # 忽略.osc目录 - if re.search("\.osc|\.git", dirpath): + if re.search("\.osc|\.git|\:Bak", dirpath): continue for filename in filenames: diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index dec8843..4588a25 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -58,7 +58,11 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False err = None if ret: logger.debug("return code {}".format(ret)) - err = p.stderr.read() - no_fmt_logger.error(err) if verbose else no_fmt_logger.debug(line) + while True: + line= p.stderr.readline() + err = line.decode("utf-8").strip() + if not err: + break + no_fmt_logger.error(err) if verbose else no_fmt_logger.debug(err) return ret, out, err if cap_err else None -- Gitee From f2e9e7359b87dff1f8a8188046ae248895992e47 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Thu, 3 Jun 2021 09:38:20 +0800 Subject: [PATCH 07/39] bug: not show err when start with blank line --- src/utils/shell_cmd.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index 4588a25..9eefc42 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -60,9 +60,9 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False logger.debug("return code {}".format(ret)) while True: line= p.stderr.readline() - err = line.decode("utf-8").strip() - if not err: + if not line: break + err = line.decode("utf-8").strip() no_fmt_logger.error(err) if verbose else no_fmt_logger.debug(err) return ret, out, err if cap_err else None -- Gitee From 9d1b3042187f493577dd9c0c54183d615c674d08 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Mon, 7 Jun 2021 17:55:19 +0800 Subject: [PATCH 08/39] ignore byte to str decode error --- src/build/extra_work.py | 4 ++-- src/utils/shell_cmd.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 3c99463..fbefcc8 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -196,8 +196,8 @@ class ExtraWork(object): logger.info("install rpms: {}".format(names)) if packages: - check_install_cmd = "sudo dnf install -y --disablerepo=* --enablerepo={}* --installroot={}" \ - " --setopt=reposdir=. {}".format(repo_name_prefix, install_root, " ".join(packages)) + check_install_cmd = "sudo dnf install -y --installroot={} --setopt=reposdir=. {}".format( + install_root, " ".join(packages)) ret, _, err = shell_cmd_live(check_install_cmd, verbose=True) if ret: logger.error("install rpms error, {}, {}".format(ret, err)) diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index 9eefc42..c68c65c 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -42,7 +42,7 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False while True: line = p.stdout.readline() if line: - line = line.decode("utf-8") + line = line.decode("utf-8", errors="ignore") line = line.strip() no_fmt_logger.info(line) if verbose else no_fmt_logger.debug(line) if cap_out and line and line != "\n": @@ -62,7 +62,7 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False line= p.stderr.readline() if not line: break - err = line.decode("utf-8").strip() + err = line.decode("utf-8", errors="ignore").strip() no_fmt_logger.error(err) if verbose else no_fmt_logger.debug(err) return ret, out, err if cap_err else None -- Gitee From adb5b1ee0c5bc9da869764ad2cda10cc9d678016 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Tue, 22 Jun 2021 16:50:51 +0800 Subject: [PATCH 09/39] add timeout of decompress file --- src/ac/common/gitee_repo.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ac/common/gitee_repo.py b/src/ac/common/gitee_repo.py index 0f4527f..996bfe4 100644 --- a/src/ac/common/gitee_repo.py +++ b/src/ac/common/gitee_repo.py @@ -103,9 +103,11 @@ class GiteeRepo(object): :return: """ if self._is_compress_zip_file(file_path): - decompress_cmd = "cd {}; unzip -d {} {}".format(self._work_dir, self._decompress_dir, file_path) + decompress_cmd = "cd {}; timeout 120s unzip -o -d {} {}".format( + self._work_dir, self._decompress_dir, file_path) elif self._is_compress_tar_file(file_path): - decompress_cmd = "cd {}; tar -C {} -xavf {}".format(self._work_dir, self._decompress_dir, file_path) + decompress_cmd = "cd {}; timeout 120s tar -C {} -xavf {}".format( + self._work_dir, self._decompress_dir, file_path) else: logger.warning("unsupport compress file: {}".format(file_path)) return False -- Gitee From d2b28bb9c0dfda43c19dd37167c46db8d37c270c Mon Sep 17 00:00:00 2001 From: Jiachen Fan Date: Fri, 25 Jun 2021 06:14:55 +0800 Subject: [PATCH 10/39] skip check and testcases need internet accessibility --- src/ac/acl/package_yaml/check_yaml.py | 3 ++- src/ac/acl/spec/check_spec.py | 3 ++- test/ac/acl/package_yaml/test_check_repo.py | 9 ++++++++ test/ac/acl/package_yaml/test_check_yaml.py | 24 +++++++++++++++++++++ 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/ac/acl/package_yaml/check_yaml.py b/src/ac/acl/package_yaml/check_yaml.py index 62f5d0b..dad80ac 100644 --- a/src/ac/acl/package_yaml/check_yaml.py +++ b/src/ac/acl/package_yaml/check_yaml.py @@ -205,5 +205,6 @@ class CheckPackageYaml(BaseCheck): def __call__(self, *args, **kwargs): logger.info("check {} yaml ...".format(self._repo)) self._yaml_changed = self.is_change_package_yaml() # yaml文件变更 进行检查 - return self.start_check_with_order("fields", "repo_domain", "repo_name", "repo") + # 因门禁系统限制外网访问权限，将涉及外网访问的检查功能check_repo暂时关闭 + return self.start_check_with_order("fields", "repo_domain", "repo_name") diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index 289969b..ca0b0ad 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -201,4 +201,5 @@ class CheckSpec(BaseCheck): self._ex_exclusive_arch() self._tbranch = kwargs.get("tbranch", None) - return self.start_check() + # 因门禁系统限制外网访问权限，将涉及外网访问的检查功能check_homepage暂时关闭 + return self.start_check_with_order("version", "patches") diff --git a/test/ac/acl/package_yaml/test_check_repo.py b/test/ac/acl/package_yaml/test_check_repo.py index 846d071..8bf40f2 100644 --- a/test/ac/acl/package_yaml/test_check_repo.py +++ b/test/ac/acl/package_yaml/test_check_repo.py @@ -29,6 +29,8 @@ from src.ac.acl.package_yaml.check_repo import ReleaseTagsFactory logging.getLogger('test_logger') +ACCESS2INTERNET = False + class TestGetReleaseTags(unittest.TestCase): TEST_YAML_DIR = { "hg": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/hg_test/hg_test.yaml"), @@ -63,6 +65,7 @@ class TestGetReleaseTags(unittest.TestCase): release_tags = ReleaseTagsFactory.get_release_tags(vc) return release_tags.get_tags(sr) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_hg_release_tags(self): release_tags = self._get_test_tags("hg") self.assertEqual(len(release_tags) > 0, True) @@ -72,14 +75,17 @@ class TestGetReleaseTags(unittest.TestCase): # release_tags = self._get_test_tags("github") # self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_git_release_tags(self): release_tags = self._get_test_tags("git") self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_gitlab_gnome_release_tags(self): release_tags = self._get_test_tags("gitlab.gnome") self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_svn_release_tags(self): release_tags = self._get_test_tags("svn") self.assertEqual(len(release_tags) > 0, True) @@ -89,14 +95,17 @@ class TestGetReleaseTags(unittest.TestCase): # release_tags = self._get_test_tags("metacpan") # self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_pypi_release_tags(self): release_tags = self._get_test_tags("pypi") self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_rubygem_release_tags(self): release_tags = self._get_test_tags("rubygem") self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_gitee_release_tags(self): release_tags = self._get_test_tags("gitee") self.assertEqual(len(release_tags) > 0, True) diff --git a/test/ac/acl/package_yaml/test_check_yaml.py b/test/ac/acl/package_yaml/test_check_yaml.py index 57f04fa..42ee752 100644 --- a/test/ac/acl/package_yaml/test_check_yaml.py +++ b/test/ac/acl/package_yaml/test_check_yaml.py @@ -37,6 +37,8 @@ from src.ac.acl.package_yaml.check_repo import HgReleaseTags, GithubReleaseTags, logging.getLogger('test_logger') +ACCESS2INTERNET = False + class TestCheckYamlField(unittest.TestCase): TEST_YAML_DIR = { @@ -107,18 +109,21 @@ class TestCheckYamlRepo(unittest.TestCase): self.cy.set_yaml = types.MethodType(set_yaml, self.cy) self.cy.set_spec = types.MethodType(set_spec, self.cy) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_none_file(self): self.cy.set_yaml(None) self.cy.check_fields() result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_NA_repo(self): self.cy.set_yaml(self.TEST_YAML_DIR["na"]) self.cy.check_fields() result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(HgReleaseTags, "get_tags") def test_hg_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["hg"]) @@ -127,6 +132,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(HgReleaseTags, "get_tags") def test_hg_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["hg"]) @@ -135,6 +141,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GithubReleaseTags, "get_tags") def test_github_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["github"]) @@ -143,6 +150,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GithubReleaseTags, "get_tags") def test_github_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["github"]) @@ -151,6 +159,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GitReleaseTags, "get_tags") def test_git_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["git"]) @@ -159,6 +168,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GitReleaseTags, "get_tags") def test_git_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["git"]) @@ -167,6 +177,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GitlabReleaseTags, "get_tags") def test_gitlab_gnome_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gitlab.gnome"]) @@ -175,6 +186,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GitlabReleaseTags, "get_tags") def test_gitlab_gnome_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gitlab.gnome"]) @@ -183,6 +195,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(SvnReleaseTags, "get_tags") def test_svn_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["svn"]) @@ -191,6 +204,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(SvnReleaseTags, "get_tags") def test_svn_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["svn"]) @@ -199,6 +213,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(MetacpanReleaseTags, "get_tags") def test_metacpan_repoo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["metacpan"]) @@ -207,6 +222,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(MetacpanReleaseTags, "get_tags") def test_metacpan_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["metacpan"]) @@ -215,6 +231,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(PypiReleaseTags, "get_tags") def test_pypi_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["pypi"]) @@ -223,6 +240,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(PypiReleaseTags, "get_tags") def test_pypi_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["pypi"]) @@ -231,6 +249,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(RubygemReleaseTags, "get_tags") def test_rubygem_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["rubygem"]) @@ -239,6 +258,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(RubygemReleaseTags, "get_tags") def test_rubygem_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["rubygem"]) @@ -247,6 +267,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GiteeReleaseTags, "get_tags") def test_gitee_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gitee"]) @@ -255,6 +276,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GiteeReleaseTags, "get_tags") def test_gitee_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gitee"]) @@ -263,6 +285,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GnuftpReleaseTags, "get_tags") def test_gnu_ftp_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gnu-ftp"]) @@ -271,6 +294,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GnuftpReleaseTags, "get_tags") def test_gnu_ftp_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gnu-ftp"]) -- Gitee From 2e8fed880283cca0464aac48af09a537211607d3 Mon Sep 17 00:00:00 2001 From: Jiachen Fan Date: Tue, 13 Jul 2021 06:23:39 +0800 Subject: [PATCH 11/39] support later version license --- src/ac/acl/package_license/check_license.py | 2 +- .../config/later_support_license.yaml | 19 +++++++++++++ src/ac/acl/package_license/package_license.py | 27 +++++++++++++++++-- test/ac/acl/license/test_check_license.py | 6 +++++ 4 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 src/ac/acl/package_license/config/later_support_license.yaml diff --git a/src/ac/acl/package_license/check_license.py b/src/ac/acl/package_license/check_license.py index 28eeaf2..6502a02 100644 --- a/src/ac/acl/package_license/check_license.py +++ b/src/ac/acl/package_license/check_license.py @@ -87,7 +87,7 @@ class CheckLicense(BaseCheck): check whether the license in spec file and in src file is same :return """ - if self._pkg_license.check_licenses_is_same(self._license_in_spec, self._license_in_src): + if self._pkg_license.check_licenses_is_same(self._license_in_spec, self._license_in_src, self._pkg_license._later_support_license): logger.info("licenses in src:{} and in spec:{} are same".format(self._license_in_src, self._license_in_spec)) return SUCCESS diff --git a/src/ac/acl/package_license/config/later_support_license.yaml b/src/ac/acl/package_license/config/later_support_license.yaml new file mode 100644 index 0000000..ff1b5e1 --- /dev/null +++ b/src/ac/acl/package_license/config/later_support_license.yaml @@ -0,0 +1,19 @@ +GPL: + versions: + - "1.0" + - "2.0" + - "3.0" +LGPL: + versions: + - "2.0" + - "2.1" + - "3.0" +AGPL: + versions: + - "1.0" + - "3.0" +GFDL: + versions: + - "1.1" + - "1.2" + - "1.3" \ No newline at end of file diff --git a/src/ac/acl/package_license/package_license.py b/src/ac/acl/package_license/package_license.py index f64f7d5..24a17c4 100644 --- a/src/ac/acl/package_license/package_license.py +++ b/src/ac/acl/package_license/package_license.py @@ -54,10 +54,14 @@ class PkgLicense(object): LICENSE_YAML_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config", "Licenses.yaml") + LATER_SUPPORT_LICENSE_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "config", + "later_support_license.yaml") def __init__(self): self._white_black_list = {} self._license_translation = {} + self._later_support_license = {} def load_config(self): """ @@ -80,6 +84,12 @@ class PkgLicense(object): except yaml.YAMLError as e: logger.exception("yaml load error: %s", str(e)) return + with open(self.LATER_SUPPORT_LICENSE_PATH, "r") as f: + try: + self._later_support_license = yaml.safe_load(f) + except yaml.YAMLError as e: + logger.exception("yaml load error: %s", str(e)) + return self._parse_tag_license(data["Software Licenses"]["Not Free Licenses"], "black") self._parse_tag_license(data["Software Licenses"]["Free Licenses"], @@ -215,9 +225,22 @@ class PkgLicense(object): return PkgLicense._decode_str(data, chardet.detect(data)["encoding"]) @staticmethod - def check_licenses_is_same(licenses_for_spec, licenses_for_source_files): + def check_licenses_is_same(licenses_for_spec, licenses_for_source_files, later_support_license): """ Check if the licenses from SPEC is the same as the licenses from LICENSE file. if same, return True. if not same return False. """ - return licenses_for_spec.issuperset(licenses_for_source_files) \ No newline at end of file + all_licenses_for_spec = set() + for license in licenses_for_spec: + if "-or-later" in license: + [l, v] = license.split("-or-later")[0].split("-", 1) + if l not in later_support_license: + all_licenses_for_spec.add(license) + continue + for version in later_support_license[l]["versions"]: + if version >= v: + all_licenses_for_spec.add(f"{l}-{version}-or-later") + all_licenses_for_spec.add(f"{l}-{version}-only") + else: + all_licenses_for_spec.add(license) + return all_licenses_for_spec.issuperset(licenses_for_source_files) \ No newline at end of file diff --git a/test/ac/acl/license/test_check_license.py b/test/ac/acl/license/test_check_license.py index ffab33c..09c3f9d 100644 --- a/test/ac/acl/license/test_check_license.py +++ b/test/ac/acl/license/test_check_license.py @@ -24,6 +24,7 @@ import types import logging.config import logging import shutil +from src.ac.acl.package_license.package_license import PkgLicense from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS from src.ac.acl.package_license.check_license import CheckLicense @@ -120,6 +121,11 @@ class TestCheckPkgLicense(unittest.TestCase): def test_check_license_same_failed(self): self._test_check_license_same("spec_src_diff", WARNING) + def test_check_license_same_later_version(self): + cl = PkgLicense() + cl.load_config() + self.assertEqual(cl.check_licenses_is_same(["GPL-1.0-or-later"], ["GPL-3.0-only"], cl._later_support_license), True) + if __name__ == "__main__": work_dir = os.getcwd() -- Gitee From fc3b8c5d09534c78c6789b268fd2e298f147a5d3 Mon Sep 17 00:00:00 2001 From: tracedeng Date: Thu, 22 Jul 2021 17:05:41 +0800 Subject: [PATCH 12/39] support obs multibuild package --- src/build/osc_build_k8s.py | 5 ++++- src/proxy/obs_proxy.py | 20 +++++++++++++------- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index d80b2ae..72e244e 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -97,7 +97,10 @@ class SinglePackageBuild(object): # osc build for repo in repos: - if not OBSProxy.build_package(project, self._package, repo["repo"], self._arch): + if repo["state"] == "excluded": + logger.info("repo {} excluded".format(repo["repo"])) + continue + if not OBSProxy.build_package(project, self._package, repo["repo"], self._arch, repo["mpac"]): logger.error("build {} ... failed".format(repo["repo"])) return 3 diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index 935d944..3c11907 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -67,8 +67,13 @@ class OBSProxy(object): rs = [] for line in out: - repo, arch, state = line.split() - rs.append({"repo": repo, "state": state}) + try: + repo, arch, state = line.split() + mpac = package + except ValueError: + repo, arch, pac, state = line.split() + mpac = pac.split(":")[-1] + rs.append({"repo": repo, "mpac": mpac, "state": state}) return rs @@ -110,22 +115,23 @@ class OBSProxy(object): return True @staticmethod - def build_package(project, package, repo, arch, debug=False): + def build_package(project, package, repo, arch, mpac, debug=False): """ build :param project: :param package: :param repo: :param arch: + :param mpac: multibuild package :param debug: :return: """ package_path = "{}/{}".format(project, package) - cmd = "cd {}; osc build {} {} {} --no-verify --clean --noservice".format( - package_path, repo, arch, "--disable-debuginfo" if not debug else "") + cmd = "cd {}; osc build {} {} {} --no-verify --clean --noservice -M {}".format( + package_path, repo, arch, "--disable-debuginfo" if not debug else "", mpac) - logger.info("osc build {} {} {} --no-verify --clean".format( - repo, arch, "--disable-debuginfo" if not debug else "")) + logger.info("osc build {} {} {} --no-verify --clean --noservice -M {}".format( + repo, arch, "--disable-debuginfo" if not debug else "", mpac)) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: -- Gitee From 99db56f5acbbe598fe15ba1dc2d08ca5705a5948 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Tue, 27 Jul 2021 18:03:54 +0800 Subject: [PATCH 13/39] only excluded raspberrypi-kernel --- src/build/osc_build_k8s.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 72e244e..f3a44db 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -97,8 +97,8 @@ class SinglePackageBuild(object): # osc build for repo in repos: - if repo["state"] == "excluded": - logger.info("repo {} excluded".format(repo["repo"])) + if repo["state"] == "excluded" and repo["mpac"] == "raspberrypi-kernel": + logger.info("repo {}:{} excluded".format(repo["repo"], repo["mpac"])) continue if not OBSProxy.build_package(project, self._package, repo["repo"], self._arch, repo["mpac"]): logger.error("build {} ... failed".format(repo["repo"])) -- Gitee From 21ffa54d53d2dd01d8e3b86608bdad601d683454 Mon Sep 17 00:00:00 2001 From: DisNight Date: Tue, 3 Aug 2021 01:27:10 +0000 Subject: [PATCH 14/39] =?UTF-8?q?=E5=A2=9E=E5=8A=A0cc0=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ac/acl/package_license/config/Licenses.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ac/acl/package_license/config/Licenses.yaml b/src/ac/acl/package_license/config/Licenses.yaml index 96f3fbf..894e300 100644 --- a/src/ac/acl/package_license/config/Licenses.yaml +++ b/src/ac/acl/package_license/config/Licenses.yaml @@ -1238,6 +1238,7 @@ Software Licenses: - alias: - Creative Commons Zero v1.0 Universal - CC0 + - CC0 1.0 Universal identifier: CC0-1.0 - alias: - TCL/TK License -- Gitee From 0447c8e1986e73bce63834466efd6d1a24a963cf Mon Sep 17 00:00:00 2001 From: georgecao Date: Thu, 5 Aug 2021 19:46:39 +0800 Subject: [PATCH 15/39] Add change logs table for openEuler CI jenkins. --- changelogs/openEuler_jenkins_changelogs.md | 23 ++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 changelogs/openEuler_jenkins_changelogs.md diff --git a/changelogs/openEuler_jenkins_changelogs.md b/changelogs/openEuler_jenkins_changelogs.md new file mode 100644 index 0000000..46f0554 --- /dev/null +++ b/changelogs/openEuler_jenkins_changelogs.md @@ -0,0 +1,23 @@ +# openEuler CI 门禁变更记录表 + +## 背景 +为规范openEuler CI 门禁配置管理，让每一次配置变更有迹可循，特制定改变更记录表，用于记录对门禁系统做出的每一次配置变更。包括但不限于Jenkins软甲升级、Jenkins插件安装、工程结构变更等。 + + +## 表格填写说明 + +**变更日期**：发起变更的具体日期，例如：“2021-08-01”； +**变更类型**：描述变更操作类型，如软件升级、插件安装等； +**变更原因**：描述该变更希望解决什么问题； +**变更说明**：描述变更的具体内容。如升级软件到什么版本，安装哪些插件； +**责任人**：该变更发起者； +**是否涉及存档**：该变更如果是软件修改或配置变更，通常会有文档或代码修改，修改前后的配置文档或代码对比需要存档，以便后续追溯； +**配置文件或代码路径**：如果“是否涉及存档”中填选“是”，则需要在当前仓库该Change Log Table的同级目录下以变更名称新建文件夹来保存相关的变更文档； + +## 变更记录 + +|序号|变更类型|变更领域|变更原因|变更说明|责任人|是否涉及存档|配置文件或代码路径| +|--|--|--|--|--|--|--|--| +| 1 | | | | | | +| 2 | | | | | | +| 3 | | | | | | -- Gitee From 23ac58cff9ea49a460bce44e594272eef9fe58d1 Mon Sep 17 00:00:00 2001 From: tan <1179903787@qq.com> Date: Wed, 28 Jul 2021 15:33:37 +0800 Subject: [PATCH 16/39] =?UTF-8?q?openeuler-jenkins=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E6=96=B0=E7=9A=84=E6=A3=80=E6=9F=A5=E9=A1=B9=EF=BC=8C=E5=AF=B9?= =?UTF-8?q?=E6=8E=A5codecheck=20check=20sca=E6=A3=80=E6=9F=A5=E9=A1=B9?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=20kunpengsecl=20=E4=BB=93=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ac/acl/openlibing/__init__.py | 17 ++++ src/ac/acl/openlibing/check_code.py | 97 +++++++++++++++++++++++ src/ac/framework/ac.py | 10 ++- src/ac/framework/ac.yaml | 7 +- src/proxy/gitee_proxy.py | 2 +- test/ac/acl/openlibing/__init__.py | 17 ++++ test/ac/acl/openlibing/test_check_code.py | 90 +++++++++++++++++++++ 7 files changed, 237 insertions(+), 3 deletions(-) create mode 100644 src/ac/acl/openlibing/__init__.py create mode 100644 src/ac/acl/openlibing/check_code.py create mode 100644 test/ac/acl/openlibing/__init__.py create mode 100644 test/ac/acl/openlibing/test_check_code.py diff --git a/src/ac/acl/openlibing/__init__.py b/src/ac/acl/openlibing/__init__.py new file mode 100644 index 0000000..60fc9c4 --- /dev/null +++ b/src/ac/acl/openlibing/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-08-03 +# Description: check code static +# ********************************************************************************** +""" \ No newline at end of file diff --git a/src/ac/acl/openlibing/check_code.py b/src/ac/acl/openlibing/check_code.py new file mode 100644 index 0000000..cd0ab58 --- /dev/null +++ b/src/ac/acl/openlibing/check_code.py @@ -0,0 +1,97 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-08-03 +# Description: check code static +# ********************************************************************************** +""" + +import logging + +from src.ac.framework.ac_base import BaseCheck +from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS +from src.proxy.requests_proxy import do_requests + +logger = logging.getLogger("ac") + + +class CheckCode(BaseCheck): + """ + code check + """ + + def __init__(self, workspace, repo, conf=None): + """ + + :param workspace: + :param repo: + :param conf: + """ + super(CheckCode, self).__init__(workspace, repo, conf) + + @staticmethod + def get_codecheck_result(pr_url, codecheck_api_url): + """ + 通过api调用codecheck + """ + data = {"pr_url": pr_url} + response_content = {} + logger.info("codecheck probably need to 3min") + rs = do_requests("get", codecheck_api_url, querystring=data, timeout=180, obj=response_content) + return rs, response_content + + def check_code(self): + """ + 开始进行codecheck检查 + """ + # 等待计算结果 + rs, response_content = self.get_codecheck_result(self._pr_url, self._codecheck_api_url) + + # 判断是否计算完成 + if rs != 0: + return SUCCESS + + if response_content.get('msg') == 'success': + """ + # 返回结果 { + "code": "200", + "msg": "success", + "data": "http://{ip}:{port}/inc/{projectId}/reports/{taskId}/detail" 一个可以看到codecheck检查结果详情的地址 + "state": "pass(通过)/no pass(不通过)" + } + """ + # 只有codecheck完成且codecheck检查的代码中存在bug，返回检查项失败的结果，以detail结尾，会显示具体的代码bug所在位置。 + if response_content.get("state") == "no pass": + logger.warning("click {} view code check detail".format(response_content.get('data'))) + return FAILED + else: + logger.error("code check failed, info :{}".format(response_content.get('msg'))) + + return SUCCESS + + def __call__(self, *args, **kwargs): + """ + 入口函数 + :param args: + :param kwargs: + :return: + """ + logger.info("check {} code ...".format(self._repo)) + logger.debug("args: {}, kwargs: {}".format(args, kwargs)) + codecheck_conf = kwargs.get("codecheck", {}) + + self._pr_url = codecheck_conf.get("pr_url", "") + self._pr_number = codecheck_conf.get("pr_number", "") + self._codecheck_api_url = codecheck_conf.get("codecheck_api_url", "") + + return self.start_check() diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index da90427..6e32ebb 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -209,6 +209,9 @@ def init_args(): default="scanoss_result", help="scanoss result output") parser.add_argument("--scanoss-repo-path", type=str, dest="scanoss_repo", help="scanoss result repo path") + parser.add_argument("--codecheck-api-url", type=str, dest="codecheck_api_url", + default="http://124.71.75.234:8384/api/openlibing/codecheck/start", help="codecheck api url") + return parser.parse_args() @@ -289,9 +292,14 @@ if "__main__" == __name__: scanoss = {"api_key": args.scanoss_api_key, "api_url": args.scanoss_api_url, "output": args.scanoss_output, "repo_path": args.scanoss_repo} + codecheck = {"pr_url": "https://gitee.com/{}/{}/pulls/{}".format(args.community, args.repo, args.pr), + "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url + } + # build ac = AC(os.path.join(os.path.dirname(os.path.realpath(__file__)), "ac.yaml"), args.community) - ac.check_all(workspace=args.workspace, repo=args.repo, dataset=dd, tbranch=args.tbranch, scanoss=scanoss) + ac.check_all(workspace=args.workspace, repo=args.repo, dataset=dd, tbranch=args.tbranch, scanoss=scanoss, + codecheck=codecheck) dd.set_attr_etime("access_control.build.etime") ac.save(args.output) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 3a12e87..9946b92 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -34,4 +34,9 @@ openeuler: hint: check_sca module: sca.check_sca entry: CheckSCA - allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform"] + allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform", "kunpengsecl"] + openlibing: + hint: check_openlibing + module: openlibing.check_code + entry: CheckCode + allow_list: ["pkgship", "kunpengsecl", "release-tools"] diff --git a/src/proxy/gitee_proxy.py b/src/proxy/gitee_proxy.py index a46659a..3e30c60 100644 --- a/src/proxy/gitee_proxy.py +++ b/src/proxy/gitee_proxy.py @@ -170,4 +170,4 @@ class GiteeProxy(object): if rs != 0: logger.warning("get last pr committer failed") - return committer[0] + return committer[0] \ No newline at end of file diff --git a/test/ac/acl/openlibing/__init__.py b/test/ac/acl/openlibing/__init__.py new file mode 100644 index 0000000..9d89286 --- /dev/null +++ b/test/ac/acl/openlibing/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-08-03 +# Description: check spec file +# ********************************************************************************** +""" \ No newline at end of file diff --git a/test/ac/acl/openlibing/test_check_code.py b/test/ac/acl/openlibing/test_check_code.py new file mode 100644 index 0000000..8f0623b --- /dev/null +++ b/test/ac/acl/openlibing/test_check_code.py @@ -0,0 +1,90 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-08-03 +# Description: check spec file +# ********************************************************************************** +""" + + +import unittest +import os +import logging.config +import logging +import shutil + +from unittest import mock +from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS +from src.ac.acl.openlibing.check_code import CheckCode + +logging.getLogger('test_logger') + + +class TestCheckCode(unittest.TestCase): + + CODECHECK = { + "codecheck_api_url": "http://124.71.75.234:8384/api/openlibing/codecheck/start", + "pr_url": "https://gitee.com/openeuler/pkgship/pulls/210", + "pr_number": "210", + "repo": "pkgship" + } + + def _test_check_code(self, predict): + cc = CheckCode('./test', self.CODECHECK.get("repo")) + self.assertEqual(cc(codecheck=self.CODECHECK), mock.Mock(return_value=predict)()) + + def _test_codecheck_api(self, return_value): + mock_fun = mock.Mock(return_value=return_value) + self.assertEqual(CheckCode.get_codecheck_result + (self.CODECHECK.get('pr_url'), self.CODECHECK.get("codecheck_api_url")), mock_fun()) + + def test_check_code_success(self): + self._test_check_code(SUCCESS) + + def test_codecheck_api_success(self): + return_value = (0, { + "code": "200", + "msg": "success", + "data": "http://124.71.75.234/inc/315fda2799c94eaf90fda86dfe1148c1/reports/" + + "90b92657bdc35b53481711ff6eac5f71/summary" + }) + self._test_codecheck_api(return_value) + + def test_codecheck_api_success_no_pr_url(self): + return_value = (0, { + "code": "400", + "msg": "pr_url is not null", + }) + self.CODECHECK['pr_url'] = "" + self._test_codecheck_api(return_value) + + def test_codecheck_api_success_no_fail(self): + return_value = (0, { + "code": "500", + "msg": "codecheck is error", + }) + self._test_codecheck_api(return_value) + + +if __name__ == "__main__": + work_dir = os.getcwd() + _ = not os.path.exists("log") and os.mkdir("log") + logger_conf_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), + "../../../../src/conf/logger.conf")) + logging.config.fileConfig(logger_conf_path) + logger = logging.getLogger("test_logger") + # Test Package License + suite = unittest.makeSuite(TestCheckCode) + unittest.TextTestRunner().run(suite) + os.chdir(work_dir) + shutil.rmtree("log") -- Gitee From 78f7df6d2b22673876d476185fb5c6f8c1f0583b Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Thu, 12 Aug 2021 10:52:59 +0800 Subject: [PATCH 17/39] support 21.09 --- src/build/osc_build_k8s.py | 3 ++- src/tools/obs_package_build_report.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index f3a44db..7a29549 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -33,13 +33,14 @@ class SinglePackageBuild(object): "openEuler-20.03-LTS": ["openEuler:20.03:LTS"], "openEuler-20.03-LTS-Next": ["openEuler:20.03:LTS:Next", "openEuler:20.03:LTS:Next:Epol"], "openEuler-EPOL-LTS": ["bringInRely"], - "openEuler-20.09": ["openEuler:20.09", "openEuler:20.09:Epol"], + "openEuler-20.09": ["openEuler:20.09", "openEuler:20.09:Epol", "openEuler:20.09:Extras"], "mkopeneuler-20.03": ["openEuler:Extras"], "openEuler-20.03-LTS-SP1": ["openEuler:20.03:LTS:SP1", "openEuler:20.03:LTS:SP1:Epol", "openEuler:20.03:LTS:SP1:Extras"], "openEuler-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2", "openEuler:20.03:LTS:SP2:Epol", "openEuler:20.03:LTS:SP2:Extras"], "openEuler-21.03": ["openEuler:21.03", "openEuler:21.03:Epol", "openEuler:21.03:Extras"], + "openEuler-21.09": ["openEuler:21.09", "openEuler:21.09:Epol", "openEuler:21.09:Extras"], "oepkg_openstack-common_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:common"], "oepkg_openstack-queens_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:queens"], "oepkg_openstack-rocky_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:rocky"], diff --git a/src/tools/obs_package_build_report.py b/src/tools/obs_package_build_report.py index 6c018bd..cdb5084 100644 --- a/src/tools/obs_package_build_report.py +++ b/src/tools/obs_package_build_report.py @@ -45,6 +45,9 @@ class ObsPackageBuildReport(object): "openEuler:21.03": "openEuler-21.03", "openEuler:21.03:Epol": "openEuler-21.03", "openEuler:21.03:Extras": "openEuler-21.03", + "openEuler:21.09": "openEuler-21.09", + "openEuler:21.09:Epol": "openEuler-21.09", + "openEuler:21.09:Extras": "openEuler-21.09", "openEuler:20.03:LTS:SP2:oepkg:openstack:common": "oepkg_openstack-common_oe-20.03-LTS-SP2", "openEuler:20.03:LTS:SP2:oepkg:openstack:queens": "oepkg_openstack-queens_oe-20.03-LTS-SP2", "openEuler:20.03:LTS:SP2:oepkg:openstack:rocky": "oepkg_openstack-rocky_oe-20.03-LTS-SP2", -- Gitee From 638255a711249828f38c597b678597764dbd16e1 Mon Sep 17 00:00:00 2001 From: georgecao Date: Thu, 12 Aug 2021 15:37:00 +0800 Subject: [PATCH 18/39] Add HTTP Request Plugin and upgrade Credentials Plugin to 2.5 version. --- changelogs/openEuler_jenkins_changelogs.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelogs/openEuler_jenkins_changelogs.md b/changelogs/openEuler_jenkins_changelogs.md index 46f0554..b1d61f7 100644 --- a/changelogs/openEuler_jenkins_changelogs.md +++ b/changelogs/openEuler_jenkins_changelogs.md @@ -16,8 +16,8 @@ ## 变更记录 -|序号|变更类型|变更领域|变更原因|变更说明|责任人|是否涉及存档|配置文件或代码路径| -|--|--|--|--|--|--|--|--| -| 1 | | | | | | +|序号|变更类型|变更原因|变更说明|责任人|是否涉及存档|配置文件或代码路径| +|--|--|--|--|--|--|--| +| 1 | 插件安装与升级 |支撑代码扫描工具落地 |因代码扫描工具scanoss需要在jenkins中安装http_request插件，所以安装插件HTTP Request Plugin 1.10版本，同时升级其依赖的插件Credentials Plugin 从2.4.1到2.5版本 |曹志、邓鹏、颜小兵 |否 | NULL | | 2 | | | | | | | 3 | | | | | | -- Gitee From 86662d2432b6d74d0680fc850ffaaf2a3834810f Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Tue, 24 Aug 2021 15:49:10 +0800 Subject: [PATCH 19/39] repo release-tools support sca --- src/ac/framework/ac.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 9946b92..b1043a4 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -21,6 +21,8 @@ src-openeuler: entry: CheckLicense sca: exclude: True + openlibing: + exclude: True openeuler: spec: exclude: True @@ -34,7 +36,7 @@ openeuler: hint: check_sca module: sca.check_sca entry: CheckSCA - allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform", "kunpengsecl"] + allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform", "kunpengsecl", "release-tools"] openlibing: hint: check_openlibing module: openlibing.check_code -- Gitee From 45aabfad95dc297b2da06dd90bd85fceed136aa3 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Sat, 18 Sep 2021 11:45:57 +0800 Subject: [PATCH 20/39] 1 build iproute use root, 2 fix scanoss upgrade compatible, 3 fix analysis spec ExclusiveArch bug --- src/ac/common/pyrpm.py | 2 +- src/build/osc_build_k8s.py | 4 +++- src/conf/deny_list.sbom | 10 +++++----- src/proxy/obs_proxy.py | 12 +++++++----- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/ac/common/pyrpm.py b/src/ac/common/pyrpm.py index 7068aa9..b958bb6 100644 --- a/src/ac/common/pyrpm.py +++ b/src/ac/common/pyrpm.py @@ -174,7 +174,7 @@ _tags = [ _NameValue("group", re_tag_compile(r"^Group\s*:\s*(\S+)")), _NameValue("url", re_tag_compile(r"^URL\s*:\s*(\S+)")), _NameValue("buildroot", re_tag_compile(r"^BuildRoot\s*:\s*(\S+)")), - _NameValue("buildarch", re_tag_compile(r"^ExclusiveArch\s*:\s*(\S+)")), + _NameValue("buildarch", re_tag_compile(r"^ExclusiveArch\s*:\s*(\S.*)")), _ListAndDict("sources", re_tag_compile(r"^(Source\d*)\s*:\s*(\S+)")), _ListAndDict("patches", re_tag_compile(r"^(Patch\d*)\s*:\s*(\S+)")), _List("build_requires", re_tag_compile(r"^BuildRequires\s*:\s*(.+)")), diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 7a29549..cc02d00 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -101,7 +101,9 @@ class SinglePackageBuild(object): if repo["state"] == "excluded" and repo["mpac"] == "raspberrypi-kernel": logger.info("repo {}:{} excluded".format(repo["repo"], repo["mpac"])) continue - if not OBSProxy.build_package(project, self._package, repo["repo"], self._arch, repo["mpac"]): + root_build = True if repo["mpac"] == "iproute" else False + if not OBSProxy.build_package( + project, self._package, repo["repo"], self._arch, repo["mpac"], root_build): logger.error("build {} ... failed".format(repo["repo"])) return 3 diff --git a/src/conf/deny_list.sbom b/src/conf/deny_list.sbom index 87e0d67..0528fad 100644 --- a/src/conf/deny_list.sbom +++ b/src/conf/deny_list.sbom @@ -2,23 +2,23 @@ "comment": "Component Denylist", "components": [ { - "publish": "openeuler", + "publisher": "openeuler", "name": "pkgship" }, { - "publish": "openeuler", + "publisher": "openeuler", "name": "openeuler-jenkins" }, { - "publish": "openeuler", + "publisher": "openeuler", "name": "stratovirt" }, { - "publish": "openeuler", + "publisher": "openeuler", "name": "secGear" }, { - "publish": "openeuler", + "publisher": "openeuler", "name": "isula-transform" } ] diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index 3c11907..cda15e7 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -115,7 +115,7 @@ class OBSProxy(object): return True @staticmethod - def build_package(project, package, repo, arch, mpac, debug=False): + def build_package(project, package, repo, arch, mpac, debug=False, root_build=False): """ build :param project: @@ -127,11 +127,13 @@ class OBSProxy(object): :return: """ package_path = "{}/{}".format(project, package) - cmd = "cd {}; osc build {} {} {} --no-verify --clean --noservice -M {}".format( - package_path, repo, arch, "--disable-debuginfo" if not debug else "", mpac) + root_opt = "--userootforbuild" if root_build else "" + debuginfo_opt = "--disable-debuginfo" if not debug else "" + cmd = "cd {}; osc build {} {} {} {} --no-verify --clean --noservice -M {}".format( + package_path, repo, arch, root_opt, debuginfo_opt, mpac) - logger.info("osc build {} {} {} --no-verify --clean --noservice -M {}".format( - repo, arch, "--disable-debuginfo" if not debug else "", mpac)) + logger.info("osc build {} {} {} {} --no-verify --clean --noservice -M {}".format( + repo, arch, root_opt, debuginfo_opt, mpac)) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: -- Gitee From c56368b9cd7002da8a7852440c2580701f7d487f Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Thu, 23 Sep 2021 09:57:53 +0800 Subject: [PATCH 21/39] fix bug: kwargs for root_build --- src/build/osc_build_k8s.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index cc02d00..d628a73 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -101,9 +101,9 @@ class SinglePackageBuild(object): if repo["state"] == "excluded" and repo["mpac"] == "raspberrypi-kernel": logger.info("repo {}:{} excluded".format(repo["repo"], repo["mpac"])) continue - root_build = True if repo["mpac"] == "iproute" else False + root_build = repo["mpac"] == "iproute" if not OBSProxy.build_package( - project, self._package, repo["repo"], self._arch, repo["mpac"], root_build): + project, self._package, repo["repo"], self._arch, repo["mpac"], root_build=root_build): logger.error("build {} ... failed".format(repo["repo"])) return 3 -- Gitee From 670e368237adf521059121721d4d9b85ed7180c3 Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Tue, 28 Sep 2021 15:09:11 +0800 Subject: [PATCH 22/39] support libaio use root build A-Ops multipath-tools need root --- src/build/osc_build_k8s.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index d628a73..ae4a991 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -50,6 +50,7 @@ class SinglePackageBuild(object): } BUILD_IGNORED_GITEE_BRANCH = ["riscv"] + PACKAGES_USE_ROOT = ["iproute", "libaio", "A-Ops", "multipath-tools"] def __init__(self, package, arch, target_branch): """ @@ -101,7 +102,7 @@ class SinglePackageBuild(object): if repo["state"] == "excluded" and repo["mpac"] == "raspberrypi-kernel": logger.info("repo {}:{} excluded".format(repo["repo"], repo["mpac"])) continue - root_build = repo["mpac"] == "iproute" + root_build = repo["mpac"] in self.PACKAGES_USE_ROOT if not OBSProxy.build_package( project, self._package, repo["repo"], self._arch, repo["mpac"], root_build=root_build): logger.error("build {} ... failed".format(repo["repo"])) -- Gitee From d2bf24adefe3231dbd026eaad958b6014464829a Mon Sep 17 00:00:00 2001 From: tushenmei Date: Mon, 11 Oct 2021 17:19:58 +0800 Subject: [PATCH 23/39] add /release-tools-dockerfile --- src/dockerfile/README.md | 5 ++++ src/dockerfile/release-tools-dockerfile | 40 +++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 src/dockerfile/release-tools-dockerfile diff --git a/src/dockerfile/README.md b/src/dockerfile/README.md index b4af737..622bee8 100644 --- a/src/dockerfile/README.md +++ b/src/dockerfile/README.md @@ -29,3 +29,8 @@ > choose an openjdk image that from stable openeuler version - swr.cn-north-4.myhuaweicloud.com/openeuler/ci/soe:{version} + +## openeuler image based on openeuler-sp2 +> install the python packages required by release-tools based on sp2 +> Dockerfile: release-tools-dockerfile +- docker image name: openeuler-sp2-release-tools \ No newline at end of file diff --git a/src/dockerfile/release-tools-dockerfile b/src/dockerfile/release-tools-dockerfile new file mode 100644 index 0000000..e4976dc --- /dev/null +++ b/src/dockerfile/release-tools-dockerfile @@ -0,0 +1,40 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +RUN set -eux; \ + yum install -y python-jenkins python3-requests python-concurrent-log-handler python3-gevent python3-marshmallow python3-pyyaml python-pandas python-xlrd python-retrying python-esdk-obs-python git + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] -- Gitee From 8a629eef935f2e7ff390c69a7f0a3fbc4215de5c Mon Sep 17 00:00:00 2001 From: tracedeng <56997132@qq.com> Date: Wed, 13 Oct 2021 09:36:39 +0800 Subject: [PATCH 24/39] fix cpio archieve error --- src/build/osc_build_k8s.py | 3 ++- src/conf/deny_list.sbom | 4 ++++ src/proxy/obs_proxy.py | 11 ++++++----- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index ae4a991..1e3e354 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -104,7 +104,8 @@ class SinglePackageBuild(object): continue root_build = repo["mpac"] in self.PACKAGES_USE_ROOT if not OBSProxy.build_package( - project, self._package, repo["repo"], self._arch, repo["mpac"], root_build=root_build): + project, self._package, repo["repo"], self._arch, repo["mpac"], + root_build=root_build, disable_cpio=True): logger.error("build {} ... failed".format(repo["repo"])) return 3 diff --git a/src/conf/deny_list.sbom b/src/conf/deny_list.sbom index 0528fad..6b793ed 100644 --- a/src/conf/deny_list.sbom +++ b/src/conf/deny_list.sbom @@ -9,6 +9,10 @@ "publisher": "openeuler", "name": "openeuler-jenkins" }, + { + "publisher": "openeuler-mirror", + "name": "openeuler-jenkins" + }, { "publisher": "openeuler", "name": "stratovirt" diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index cda15e7..6724748 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -115,7 +115,7 @@ class OBSProxy(object): return True @staticmethod - def build_package(project, package, repo, arch, mpac, debug=False, root_build=False): + def build_package(project, package, repo, arch, mpac, debug=False, root_build=False, disable_cpio=False): """ build :param project: @@ -129,11 +129,12 @@ class OBSProxy(object): package_path = "{}/{}".format(project, package) root_opt = "--userootforbuild" if root_build else "" debuginfo_opt = "--disable-debuginfo" if not debug else "" - cmd = "cd {}; osc build {} {} {} {} --no-verify --clean --noservice -M {}".format( - package_path, repo, arch, root_opt, debuginfo_opt, mpac) + disable_cpio_bulk = "--disable-cpio-bulk-download" if disable_cpio else "" + cmd = "cd {}; osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + package_path, repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) - logger.info("osc build {} {} {} {} --no-verify --clean --noservice -M {}".format( - repo, arch, root_opt, debuginfo_opt, mpac)) + logger.info("osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac)) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: -- Gitee From 4d7244bba4726a6648def9934ac988107e4aee63 Mon Sep 17 00:00:00 2001 From: tan <1179903787@qq.com> Date: Wed, 13 Oct 2021 18:42:36 +0800 Subject: [PATCH 25/39] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E9=A1=B9=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ac/framework/ac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 9946b92..36d6e13 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -36,7 +36,7 @@ openeuler: entry: CheckSCA allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform", "kunpengsecl"] openlibing: - hint: check_openlibing + hint: code_check module: openlibing.check_code entry: CheckCode allow_list: ["pkgship", "kunpengsecl", "release-tools"] -- Gitee From 394769327e20a00d43f9dc602b592f148769a7a8 Mon Sep 17 00:00:00 2001 From: gaodi12 Date: Thu, 14 Oct 2021 15:59:59 +0800 Subject: [PATCH 26/39] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E9=A1=B9=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ac/framework/ac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 36d6e13..7ff2457 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -36,7 +36,7 @@ openeuler: entry: CheckSCA allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform", "kunpengsecl"] openlibing: - hint: code_check + hint: code module: openlibing.check_code entry: CheckCode allow_list: ["pkgship", "kunpengsecl", "release-tools"] -- Gitee From 5ffa259960a1bec56731d92b5122862a9ecbf6f3 Mon Sep 17 00:00:00 2001 From: hht8 Date: Fri, 22 Oct 2021 15:35:25 +0800 Subject: [PATCH 27/39] Add some packages to deny_list --- src/conf/deny_list.sbom | 196 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 196 insertions(+) diff --git a/src/conf/deny_list.sbom b/src/conf/deny_list.sbom index 6b793ed..fd3a23e 100644 --- a/src/conf/deny_list.sbom +++ b/src/conf/deny_list.sbom @@ -5,6 +5,14 @@ "publisher": "openeuler", "name": "pkgship" }, + { + "publisher": "openeuler-mirror", + "name": "pkgship" + }, + { + "publisher": "src-openeuler", + "name": "pkgship" + }, { "publisher": "openeuler", "name": "openeuler-jenkins" @@ -13,17 +21,205 @@ "publisher": "openeuler-mirror", "name": "openeuler-jenkins" }, + { + "publisher": "src-openeuler", + "name": "openeuler-jenkins" + }, { "publisher": "openeuler", "name": "stratovirt" }, + { + "publisher": "openeuler-mirror", + "name": "stratovirt" + }, + { + "publisher": "src-openeuler", + "name": "stratovirt" + }, { "publisher": "openeuler", "name": "secGear" }, + { + "publisher": "openeuler-mirror", + "name": "secGear" + }, + { + "publisher": "src-openeuler", + "name": "secGear" + }, { "publisher": "openeuler", "name": "isula-transform" + }, + { + "publisher": "openeuler-mirror", + "name": "isula-transform" + }, + { + "publisher": "src-openeuler", + "name": "isula-transform" + }, + { + "publisher": "openeuler-mirror", + "name": "kernel" + }, + { + "publisher": "openeuler-mirror", + "name": "async-libfuse" + }, + { + "publisher": "openeuler-mirror", + "name": "authz" + }, + { + "publisher": "openeuler-mirror", + "name": "clibcni" + }, + { + "publisher": "openeuler-mirror", + "name": "isula-build" + }, + { + "publisher": "openeuler-mirror", + "name": "iSulad" + }, + { + "publisher": "openeuler-mirror", + "name": "iSulad-img" + }, + { + "publisher": "openeuler-mirror", + "name": "iSula-libutils" + }, + { + "publisher": "openeuler-mirror", + "name": "isula-transform" + }, + { + "publisher": "openeuler-mirror", + "name": "lcr" + }, + { + "publisher": "openeuler-mirror", + "name": "lib-shim-v2" + }, + { + "publisher": "openeuler-mirror", + "name": "lxcfs-tools" + }, + { + "publisher": "openeuler-mirror", + "name": "syscontainer-tools" + }, + { + "publisher": "openeuler-mirror", + "name": "A-Tune" + }, + { + "publisher": "openeuler-mirror", + "name": "A-Tune-Collector" + }, + { + "publisher": "openeuler-mirror", + "name": "A-Tune-UI" + }, + { + "publisher": "openeuler-mirror", + "name": "prefetch_tuning" + }, + { + "publisher": "openeuler-mirror", + "name": "wisdom-advisor" + }, + { + "publisher": "openeuler-mirror", + "name": "qemu" + }, + { + "publisher": "openeuler-mirror", + "name": "vmtop" + }, + { + "publisher": "src-openeuler", + "name": "kernel" + }, + { + "publisher": "src-openeuler", + "name": "async-libfuse" + }, + { + "publisher": "src-openeuler", + "name": "authz" + }, + { + "publisher": "src-openeuler", + "name": "clibcni" + }, + { + "publisher": "src-openeuler", + "name": "isula-build" + }, + { + "publisher": "src-openeuler", + "name": "iSulad" + }, + { + "publisher": "src-openeuler", + "name": "iSulad-img" + }, + { + "publisher": "src-openeuler", + "name": "iSula-libutils" + }, + { + "publisher": "src-openeuler", + "name": "isula-transform" + }, + { + "publisher": "src-openeuler", + "name": "lcr" + }, + { + "publisher": "src-openeuler", + "name": "lib-shim-v2" + }, + { + "publisher": "src-openeuler", + "name": "lxcfs-tools" + }, + { + "publisher": "src-openeuler", + "name": "syscontainer-tools" + }, + { + "publisher": "src-openeuler", + "name": "A-Tune" + }, + { + "publisher": "src-openeuler", + "name": "A-Tune-Collector" + }, + { + "publisher": "src-openeuler", + "name": "A-Tune-UI" + }, + { + "publisher": "src-openeuler", + "name": "prefetch_tuning" + }, + { + "publisher": "src-openeuler", + "name": "wisdom-advisor" + }, + { + "publisher": "src-openeuler", + "name": "qemu" + }, + { + "publisher": "src-openeuler", + "name": "vmtop" } ] } -- Gitee From 41f28a31573075300097a90fd18f99f56d36e18c Mon Sep 17 00:00:00 2001 From: hcy1012 <690765764@qq.com> Date: Wed, 27 Oct 2021 08:30:55 +0000 Subject: [PATCH 28/39] =?UTF-8?q?update=20README.md.=20=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E8=8B=B1=E6=96=87=E8=AF=AD=E6=B3=95=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b9ed15e..70be875 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Introduction -This repository is used to store the jenkins scripts in openEuler Community. +This repository is used to store the Jenkins scripts in the openEuler community. ## License -- Gitee From 109dadd6baddcc611e4e76a23914481aa20d9b2e Mon Sep 17 00:00:00 2001 From: hcy1012 <690765764@qq.com> Date: Wed, 27 Oct 2021 08:32:22 +0000 Subject: [PATCH 29/39] =?UTF-8?q?add=20README-zh-cn.md.=20=E6=96=B0?= =?UTF-8?q?=E5=A2=9E=E4=B8=AD=E6=96=87=E7=89=88=E6=9C=ACREADME?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README-zh-cn.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 README-zh-cn.md diff --git a/README-zh-cn.md b/README-zh-cn.md new file mode 100644 index 0000000..c9525c3 --- /dev/null +++ b/README-zh-cn.md @@ -0,0 +1,9 @@ +# openeuler-jenkins + +## 简介 + +此仓库用来存放openEuler社区的Jenkins脚本。 + +## 许可证 + +详情请参考[LICENSE](https://gitee.com/openeuler/openeuler-jenkins/blob/ac397ce3e078937c700df6fb8de0e1b065ee4218/LICENSE)文件。 \ No newline at end of file -- Gitee From 4ba3153b7a5e64fab693df5360724afcf7968a84 Mon Sep 17 00:00:00 2001 From: hht8 Date: Thu, 14 Oct 2021 20:17:56 +0800 Subject: [PATCH 30/39] scanoss log optimization --- src/ac/acl/sca/check_sca.py | 66 +++++++++---------------------------- src/ac/framework/ac.py | 7 +--- src/requirements | 2 +- 3 files changed, 18 insertions(+), 57 deletions(-) diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index da04991..5d5be07 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -17,6 +17,7 @@ import os import shutil import logging +import json from src.proxy.git_proxy import GitProxy from src.ac.framework.ac_base import BaseCheck @@ -39,50 +40,22 @@ class CheckSCA(BaseCheck): """ super(CheckSCA, self).__init__(workspace, repo, conf) - self._work_diff_dir = os.path.join(workspace, "diff") # 目标目录，保存变更了的代码 - - def copy_diff_files_to_dest(self, files): - """ - 拷贝所有diff文件到目标目录 - :param files: 文件列表 - :return: - """ - for filepath in files: - try: - shutil.copy(os.path.join(self._work_dir, filepath), self._work_diff_dir) - except IOError: - logger.exception("copy {} to {} exception".format(filepath, self._work_diff_dir)) - - def save_scanoss_result(self, html): - """ - 保存结果到本地 - :param html: scanoss 结果，html格式 - :return: - """ - with open(self._scanoss_result_output, "w") as f: - f.write(html) - def check_scanoss(self): """ - scanoss工具检查代码片段引用 - https://osskb.org - https://github.com/scanoss/scanner.py - :return: + Obtain scanoss logs and result """ - gp = GitProxy(self._work_dir) - diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") - logger.debug("diff files: {}".format(diff_files)) - - self.copy_diff_files_to_dest(diff_files) - - blacklist_sbom = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../../../conf/deny_list.sbom")) - scan = ScanOSS(self._scanoss_api_key, self._scanoss_api_url, blacklist_sbom) - result = scan.scan(self._work_diff_dir) - + # Describes the reportUrl result jenkinsJobName jenkinsBuildNum prNo repoUrl of scanoss + try: + with open(self._scanoss_result_output, 'r') as f: + result_dirt = json.load(f) + except IOError: + logger.error("{} not found, make sure this file exists".format(self._scanoss_result_output)) + return FAILED + + result = result_dirt.get('result') + # 保存详细结果到web server - if not result: - self.save_scanoss_result(scan.html) - logger.warning("click {} view scanoss detail".format(self._scanoss_result_repo_path)) + logger.warning("click {} view scanoss detail".format(result_dirt.get('reportUrl'))) return SUCCESS if result else FAILED @@ -97,13 +70,6 @@ class CheckSCA(BaseCheck): logger.debug("args: {}, kwargs: {}".format(args, kwargs)) scanoss_conf = kwargs.get("scanoss", {}) - self._scanoss_api_key = scanoss_conf.get("api_key", "") - self._scanoss_api_url = scanoss_conf.get("api_url", "https://osskb.org/api/scan/direct") - self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") # 保存结果到本地文件 - self._scanoss_result_repo_path = scanoss_conf.get("repo_path", "-lost linker-") # 保存结果到web server的路径 - - _ = not os.path.exists(self._work_diff_dir) and os.mkdir(self._work_diff_dir) - try: - return self.start_check() - finally: - shutil.rmtree(self._work_diff_dir) + self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") + + return self.start_check() diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index 6e32ebb..7789bca 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -202,12 +202,8 @@ def init_args(): parser.add_argument("-l", type=str, dest="trigger_link", help="job trigger link") # scanoss - parser.add_argument("--scanoss-api-key", type=str, dest="scanoss_api_key", help="scanoss api key") - parser.add_argument("--scanoss-api-url", type=str, dest="scanoss_api_url", - default="https://osskb.org/api/scan/direct", help="scanoss api url") parser.add_argument("--scanoss-output", type=str, dest="scanoss_output", default="scanoss_result", help="scanoss result output") - parser.add_argument("--scanoss-repo-path", type=str, dest="scanoss_repo", help="scanoss result repo path") parser.add_argument("--codecheck-api-url", type=str, dest="codecheck_api_url", default="http://124.71.75.234:8384/api/openlibing/codecheck/start", help="codecheck api url") @@ -289,8 +285,7 @@ if "__main__" == __name__: gp.create_tags_of_pr(args.pr, "ci_processing") # scanoss conf - scanoss = {"api_key": args.scanoss_api_key, "api_url": args.scanoss_api_url, - "output": args.scanoss_output, "repo_path": args.scanoss_repo} + scanoss = {"output": args.scanoss_output} codecheck = {"pr_url": "https://gitee.com/{}/{}/pulls/{}".format(args.community, args.repo, args.pr), "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url diff --git a/src/requirements b/src/requirements index d3fd573..7d34070 100644 --- a/src/requirements +++ b/src/requirements @@ -11,4 +11,4 @@ chardet kafka-python elasticsearch retrying -scanoss-scanner +scanoss -- Gitee From a7091a4c7b0aa5ed8b8cf09287e8bc14c3695d2c Mon Sep 17 00:00:00 2001 From: gaodi12 Date: Wed, 3 Nov 2021 16:47:05 +0800 Subject: [PATCH 31/39] =?UTF-8?q?codecheck=20=E4=BF=AE=E6=94=B9=E8=AE=BF?= =?UTF-8?q?=E9=97=AE=E5=9C=B0=E5=9D=80=EF=BC=8C=E4=BF=AE=E6=94=B9=E6=A3=80?= =?UTF-8?q?=E6=9F=A5=E9=A1=B9=E9=80=BB=E8=BE=91=EF=BC=8C=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?token=E9=AA=8C=E8=AF=81=E5=8A=9F=E8=83=BD=E3=80=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ac/acl/openlibing/check_code.py | 45 +++++++++++++++++++++++++---- src/ac/framework/ac.py | 5 ++-- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/ac/acl/openlibing/check_code.py b/src/ac/acl/openlibing/check_code.py index cd0ab58..cc0b955 100644 --- a/src/ac/acl/openlibing/check_code.py +++ b/src/ac/acl/openlibing/check_code.py @@ -17,6 +17,7 @@ """ import logging +import time from src.ac.framework.ac_base import BaseCheck from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS @@ -40,14 +41,45 @@ class CheckCode(BaseCheck): super(CheckCode, self).__init__(workspace, repo, conf) @staticmethod - def get_codecheck_result(pr_url, codecheck_api_url): + def get_codecheck_result(pr_url, codecheck_api_url, codecheck_api_key): """ 通过api调用codecheck """ - data = {"pr_url": pr_url} + # get codecheck Api Token + codecheck_token_api_url = '{}/token/{}'.format(codecheck_api_url, codecheck_api_key) + token_resp = {} + rs = do_requests("get", codecheck_token_api_url, obj=token_resp) + if rs != 0 or token_resp.get("code", "") != "200": + logger.error("get dynamic token failed") + return 'false', {} + + token = token_resp.get("data") + data = {"pr_url": pr_url, "token": token} response_content = {} + # 创建codecheck检查任务 + codecheck_task_api_url = "{}/task".format(codecheck_api_url) + rs = do_requests("get", codecheck_task_api_url, querystring=data, obj=response_content) + if rs != 0 or response_content.get('code', '') != '200': + logger.error("create codecheck task failed; {}".format(response_content.get('msg', ''))) + return 'false', {} + + uuid = response_content.get('uuid') + task_id = response_content.get('task_id') + data = {"uuid": uuid, "token": token} + codecheck_status_api_url = '{}/{}/status'.format(codecheck_api_url, task_id) + current_time = 0 logger.info("codecheck probably need to 3min") - rs = do_requests("get", codecheck_api_url, querystring=data, timeout=180, obj=response_content) + # 定时3min + while current_time < 180: + time.sleep(10) + response_content = {} + # 检查codecheck任务的执行状态 + rs = do_requests("get", codecheck_status_api_url, querystring=data, obj=response_content) + if rs == 0 and response_content.get('code') == '100': + current_time = current_time + 10 + continue + else: + break return rs, response_content def check_code(self): @@ -55,7 +87,7 @@ class CheckCode(BaseCheck): 开始进行codecheck检查 """ # 等待计算结果 - rs, response_content = self.get_codecheck_result(self._pr_url, self._codecheck_api_url) + rs, response_content = self.get_codecheck_result(self._pr_url, self._codecheck_api_url, self._codecheck_api_key) # 判断是否计算完成 if rs != 0: @@ -70,12 +102,12 @@ class CheckCode(BaseCheck): "state": "pass(通过)/no pass(不通过)" } """ + logger.warning("click {} view code check detail".format(response_content.get('data'))) # 只有codecheck完成且codecheck检查的代码中存在bug，返回检查项失败的结果，以detail结尾，会显示具体的代码bug所在位置。 if response_content.get("state") == "no pass": - logger.warning("click {} view code check detail".format(response_content.get('data'))) return FAILED else: - logger.error("code check failed, info :{}".format(response_content.get('msg'))) + logger.error("code check failed, info : {}".format(response_content.get('msg'))) return SUCCESS @@ -93,5 +125,6 @@ class CheckCode(BaseCheck): self._pr_url = codecheck_conf.get("pr_url", "") self._pr_number = codecheck_conf.get("pr_number", "") self._codecheck_api_url = codecheck_conf.get("codecheck_api_url", "") + self._codecheck_api_key = codecheck_conf.get('codecheck_api_key', "") return self.start_check() diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index 6e32ebb..f5387d2 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -209,8 +209,9 @@ def init_args(): default="scanoss_result", help="scanoss result output") parser.add_argument("--scanoss-repo-path", type=str, dest="scanoss_repo", help="scanoss result repo path") + parser.add_argument("--codecheck-api-key", type=str, dest="codecheck_api_key", help="codecheck api key") parser.add_argument("--codecheck-api-url", type=str, dest="codecheck_api_url", - default="http://124.71.75.234:8384/api/openlibing/codecheck/start", help="codecheck api url") + default="https://majun.osinfra.cn:8384/api/openlibing/codecheck", help="codecheck api url") return parser.parse_args() @@ -293,7 +294,7 @@ if "__main__" == __name__: "output": args.scanoss_output, "repo_path": args.scanoss_repo} codecheck = {"pr_url": "https://gitee.com/{}/{}/pulls/{}".format(args.community, args.repo, args.pr), - "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url + "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url, "codecheck_api_key": args.codecheck_api_key } # build -- Gitee From 033e09825cd78a938e6fa5fd987e8d1733679e81 Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Mon, 15 Nov 2021 11:31:41 +0800 Subject: [PATCH 32/39] support branch: SP3 --- openeuler-jenkins | 1 + src/build/osc_build_k8s.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) create mode 160000 openeuler-jenkins diff --git a/openeuler-jenkins b/openeuler-jenkins new file mode 160000 index 0000000..3328552 --- /dev/null +++ b/openeuler-jenkins @@ -0,0 +1 @@ +Subproject commit 3328552eb925eb44dc632d69cbca8e213ba3dae5 diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 1e3e354..e28ba34 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -46,7 +46,8 @@ class SinglePackageBuild(object): "oepkg_openstack-rocky_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:rocky"], "oepkg_openstack-common_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:common"], "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"], - "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"] + "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"], + "openEuler-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3", "openEuler:20.03:LTS:SP3:Epol"] } BUILD_IGNORED_GITEE_BRANCH = ["riscv"] -- Gitee From 50612713c1fff80b9e5be9546932f7898899f592 Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Mon, 15 Nov 2021 14:25:40 +0800 Subject: [PATCH 33/39] remove extra empty folder from last PR --- openeuler-jenkins | 1 - 1 file changed, 1 deletion(-) delete mode 160000 openeuler-jenkins diff --git a/openeuler-jenkins b/openeuler-jenkins deleted file mode 160000 index 3328552..0000000 --- a/openeuler-jenkins +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 3328552eb925eb44dc632d69cbca8e213ba3dae5 -- Gitee From c58fc1b7b40a6aa14ab0efae82591e8130a6d324 Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Mon, 15 Nov 2021 17:17:58 +0800 Subject: [PATCH 34/39] support select which spec files to build --- src/build/osc_build_k8s.py | 13 ++++++++----- src/proxy/obs_proxy.py | 11 ++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index e28ba34..dd9b32e 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -71,11 +71,12 @@ class SinglePackageBuild(object): """ return OBSProxy.list_repos_of_arch(project, self._package, self._arch, show_exclude=True) - def build_obs_repos(self, project, repos, work_dir, code_dir): + def build_obs_repos(self, project, repos, spec, work_dir, code_dir): """ build :param project: 项目名 :param repos: obs repo + :param spec: 指定spec文件 :param code_dir: 码云代码在本地路径 :param work_dir: :return: @@ -105,7 +106,7 @@ class SinglePackageBuild(object): continue root_build = repo["mpac"] in self.PACKAGES_USE_ROOT if not OBSProxy.build_package( - project, self._package, repo["repo"], self._arch, repo["mpac"], + project, self._package, repo["repo"], self._arch, spec, repo["mpac"], root_build=root_build, disable_cpio=True): logger.error("build {} ... failed".format(repo["repo"])) return 3 @@ -193,9 +194,10 @@ class SinglePackageBuild(object): return True - def build(self, work_dir, code_dir): + def build(self, spec, work_dir, code_dir): """ 入口 + :param spec: 指定spec文件 :param work_dir: obs工作目录 :param code_dir: 代码目录 :return: @@ -219,7 +221,7 @@ class SinglePackageBuild(object): logger.debug("build obs repos: {}".format(obs_repos)) has_any_repo_build = True - ret = self.build_obs_repos(project, obs_repos, work_dir, code_dir) + ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) if ret > 0: logger.debug("build run return {}".format(ret)) logger.error("build {} {} {} ... {}".format(project, self._package, self._arch, "failed")) @@ -254,6 +256,7 @@ def init_args(): parser.add_argument("-t", type=str, dest="account", help="gitee account") parser.add_argument("-o", type=str, dest="owner", default="src-openeuler", help="gitee owner") + parser.add_argument("--spec", type=str, dest="spec", default="", help="spec files") return parser.parse_args() @@ -316,7 +319,7 @@ if "__main__" == __name__: dd.set_attr_stime("spb.build.stime") spb = SinglePackageBuild(args.package, args.arch, args.branch) - rs = spb.build(args.workspace, args.code) + rs = spb.build(args.spec, args.workspace, args.code) dd.set_attr("spb.build.result", "failed" if rs else "successful") dd.set_attr_etime("spb.build.etime") diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index 6724748..9ccabc5 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -115,13 +115,14 @@ class OBSProxy(object): return True @staticmethod - def build_package(project, package, repo, arch, mpac, debug=False, root_build=False, disable_cpio=False): + def build_package(project, package, repo, arch, spec, mpac, debug=False, root_build=False, disable_cpio=False): """ build :param project: :param package: :param repo: :param arch: + :param spec: :param mpac: multibuild package :param debug: :return: @@ -130,11 +131,11 @@ class OBSProxy(object): root_opt = "--userootforbuild" if root_build else "" debuginfo_opt = "--disable-debuginfo" if not debug else "" disable_cpio_bulk = "--disable-cpio-bulk-download" if disable_cpio else "" - cmd = "cd {}; osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( - package_path, repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) + cmd = "cd {}; osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + package_path, repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) - logger.info("osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( - repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac)) + logger.info("osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac)) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: -- Gitee From b32a610f22fb9ce71fb5e82cfc0ba522cc921d34 Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Thu, 18 Nov 2021 23:10:10 +0800 Subject: [PATCH 35/39] solve logging-format-interolation(W1202) --- src/ac/acl/code/check_code_style.py | 18 +++++----- src/ac/acl/openlibing/check_code.py | 10 +++--- src/ac/acl/package_license/check_license.py | 13 +++---- src/ac/acl/package_yaml/check_repo.py | 34 +++++++++--------- src/ac/acl/package_yaml/check_yaml.py | 20 +++++------ src/ac/acl/sca/check_sca.py | 8 ++--- src/ac/acl/spec/check_spec.py | 24 ++++++------- src/ac/common/gitee_repo.py | 18 +++++----- src/ac/common/linter.py | 12 +++---- src/ac/common/rpm_spec_adapter.py | 8 ++--- src/ac/common/scanoss.py | 10 +++--- src/ac/framework/ac.py | 38 ++++++++++----------- src/ac/framework/ac_base.py | 18 +++++----- src/build/extra_work.py | 28 +++++++-------- src/build/gitee_comment.py | 20 +++++------ src/build/obs_repo_source.py | 12 +++---- src/build/osc_build_k8s.py | 30 ++++++++-------- src/jobs/jenkins_job.py | 18 +++++----- src/jobs/obs_meta_strategy.py | 10 +++--- src/jobs/repo_mapping.py | 10 +++--- src/proxy/es_proxy.py | 12 +++---- src/proxy/git_proxy.py | 26 +++++++------- src/proxy/gitee_proxy.py | 14 ++++---- src/proxy/jenkins_proxy.py | 28 +++++++-------- src/proxy/kafka_proxy.py | 4 +-- src/proxy/obs_proxy.py | 18 +++++----- src/proxy/requests_proxy.py | 12 +++---- src/tools/obs_package_build_history.py | 8 ++--- src/tools/obs_package_build_report.py | 6 ++-- src/utils/shell_cmd.py | 16 ++++++--- test/ac/acl/package_yaml/test_check_repo.py | 4 +-- 31 files changed, 257 insertions(+), 250 deletions(-) diff --git a/src/ac/acl/code/check_code_style.py b/src/ac/acl/code/check_code_style.py index cb574b5..7dde416 100644 --- a/src/ac/acl/code/check_code_style.py +++ b/src/ac/acl/code/check_code_style.py @@ -68,7 +68,7 @@ class CheckCodeStyle(BaseCheck): """ gp = GitProxy(self._work_dir) diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") - logger.debug("diff files: {}".format(diff_files)) + logger.debug("diff files: %s", diff_files) diff_code_files = [] # 仓库中变更的代码文件 diff_patch_code_files = [] # patch内的代码文件 @@ -77,7 +77,7 @@ class CheckCodeStyle(BaseCheck): diff_code_files.append(diff_file) elif GiteeRepo.is_patch_file(diff_file): patch_dir = self._gr.patch_dir_mapping.get(diff_file) - logger.debug("diff patch {} apply at dir {}".format(diff_file, patch_dir)) + logger.debug("diff patch %s apply at dir %s", diff_file, patch_dir) if patch_dir is not None: files_in_patch = gp.extract_files_path_of_patch(diff_file) patch_code_files = [os.path.join(patch_dir, file_in_patch) @@ -88,13 +88,13 @@ class CheckCodeStyle(BaseCheck): for code_file in patch_code_files if os.path.exists(code_file)]) - logger.debug("diff code files: {}".format(diff_code_files)) - logger.debug("diff patch code files: {}".format(diff_patch_code_files)) + logger.debug("diff code files: %s", diff_code_files) + logger.debug("diff patch code files: %s", diff_patch_code_files) rs_1 = self.check_file_under_work_dir(diff_code_files) - logger.debug("check_file_under_work_dir: {}".format(rs_1)) + logger.debug("check_file_under_work_dir: %s", rs_1) rs_2 = self.check_files_inner_patch(diff_patch_code_files) - logger.debug("check_files_inner_patch: {}".format(rs_2)) + logger.debug("check_files_inner_patch: %s", rs_2) return rs_1 + rs_2 @@ -131,10 +131,10 @@ class CheckCodeStyle(BaseCheck): elif GiteeRepo.is_c_cplusplus_file(file_path): rs = LinterCheck.check_c_cplusplus(file_path) else: - logger.error("error when arrive here, unsupport file {}".format(file_path)) + logger.error("error when arrive here, unsupport file %s", file_path) return SUCCESS - logger.info("Linter: {:<40} {}".format(file_path, rs)) + logger.info("Linter: %s %s", file_path, rs) if rs.get("F", 0) > 0: return FAILED @@ -150,7 +150,7 @@ class CheckCodeStyle(BaseCheck): :param kwargs: :return: """ - logger.info("check {} repo ...".format(self._repo)) + logger.info("check %s repo ...", self._repo) _ = not os.path.exists(self._work_tar_dir) and os.mkdir(self._work_tar_dir) try: diff --git a/src/ac/acl/openlibing/check_code.py b/src/ac/acl/openlibing/check_code.py index cc0b955..d058509 100644 --- a/src/ac/acl/openlibing/check_code.py +++ b/src/ac/acl/openlibing/check_code.py @@ -60,7 +60,7 @@ class CheckCode(BaseCheck): codecheck_task_api_url = "{}/task".format(codecheck_api_url) rs = do_requests("get", codecheck_task_api_url, querystring=data, obj=response_content) if rs != 0 or response_content.get('code', '') != '200': - logger.error("create codecheck task failed; {}".format(response_content.get('msg', ''))) + logger.error("create codecheck task failed; %s", response_content.get('msg', '')) return 'false', {} uuid = response_content.get('uuid') @@ -102,12 +102,12 @@ class CheckCode(BaseCheck): "state": "pass(通过)/no pass(不通过)" } """ - logger.warning("click {} view code check detail".format(response_content.get('data'))) + logger.warning("click %s view code check detail", response_content.get('data')) # 只有codecheck完成且codecheck检查的代码中存在bug，返回检查项失败的结果，以detail结尾，会显示具体的代码bug所在位置。 if response_content.get("state") == "no pass": return FAILED else: - logger.error("code check failed, info : {}".format(response_content.get('msg'))) + logger.error("code check failed, info : %s", response_content.get('msg')) return SUCCESS @@ -118,8 +118,8 @@ class CheckCode(BaseCheck): :param kwargs: :return: """ - logger.info("check {} code ...".format(self._repo)) - logger.debug("args: {}, kwargs: {}".format(args, kwargs)) + logger.info("check %s code ...", self._repo) + logger.debug("args: %s, kwargs: %s", args, kwargs) codecheck_conf = kwargs.get("codecheck", {}) self._pr_url = codecheck_conf.get("pr_url", "") diff --git a/src/ac/acl/package_license/check_license.py b/src/ac/acl/package_license/check_license.py index 6502a02..669c139 100644 --- a/src/ac/acl/package_license/check_license.py +++ b/src/ac/acl/package_license/check_license.py @@ -87,13 +87,14 @@ class CheckLicense(BaseCheck): check whether the license in spec file and in src file is same :return """ - if self._pkg_license.check_licenses_is_same(self._license_in_spec, self._license_in_src, self._pkg_license._later_support_license): - logger.info("licenses in src:{} and in spec:{} are same".format(self._license_in_src, - self._license_in_spec)) + if self._pkg_license.check_licenses_is_same(self._license_in_spec, self._license_in_src, + self._pkg_license._later_support_license): + logger.info("licenses in src:%s and in spec:%s are same", self._license_in_src, + self._license_in_spec) return SUCCESS else: - logger.error("licenses in src:{} and in spec:{} are not same".format(self._license_in_src, - self._license_in_spec)) + logger.error("licenses in src:%s and in spec:%s are not same", self._license_in_src, + self._license_in_spec) return WARNING def __call__(self, *args, **kwargs): @@ -103,7 +104,7 @@ class CheckLicense(BaseCheck): :param kwargs: :return: """ - logger.info("check {} license ...".format(self._repo)) + logger.info("check %s license ...", self._repo) _ = not os.path.exists(self._work_tar_dir) and os.mkdir(self._work_tar_dir) self._gr.decompress_all() # decompress all compressed file into work_tar_dir diff --git a/src/ac/acl/package_yaml/check_repo.py b/src/ac/acl/package_yaml/check_repo.py index 839fd0b..11cdd96 100644 --- a/src/ac/acl/package_yaml/check_repo.py +++ b/src/ac/acl/package_yaml/check_repo.py @@ -72,7 +72,7 @@ class DefaultReleaseTags(AbsReleaseTags): 通过url获取上游社区的release tags return: list """ - logging.info("unsupported version control: {}".format(self.version_control)) + logging.info("unsupported version control: %s", self.version_control) return [] @@ -118,7 +118,7 @@ class HttpReleaseTagsMixin(object): response = requests.get(url, headers=headers, timeout=timeout) need_redirect, new_url, cookies = self.get_redirect_resp(url, response) if tldextract.extract(url).domain != tldextract.extract(new_url).domain: # 判断域名是否一致 预防csrf攻击 - logging.warning("domain of redirection link is different: {}".format(new_url)) + logging.warning("domain of redirection link is different: %s", new_url) return "" if need_redirect: cookie_dict = {} @@ -128,13 +128,13 @@ class HttpReleaseTagsMixin(object): url = new_url response = requests.get(url, headers=headers, cookies=cookie_dict, timeout=timeout) except requests.exceptions.SSLError as e: - logging.warning("requests {} ssl exception, {}".format(url, e)) + logging.warning("requests %s ssl exception, %s", url, e) return "" except requests.exceptions.Timeout as e: logging.warning("requests timeout") return "" except requests.exceptions.RequestException as e: - logging.warning("requests exception, {}".format(e)) + logging.warning("requests exception, %s", e) return "" return response @@ -156,7 +156,7 @@ class HgReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -170,7 +170,7 @@ class HgReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): temp_tags.sort(reverse=True, key=lambda x: x["date"][0]) release_tags = [tag["tag"] for tag in temp_tags] except Exception as e: - logging.error("exception, {}".format(e)) + logging.error("exception, %s", e) return [] return release_tags @@ -192,7 +192,7 @@ class HgRawReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -220,7 +220,7 @@ class MetacpanReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -258,7 +258,7 @@ class PypiReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -267,7 +267,7 @@ class PypiReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): tags_json = response.json() release_tags = [tag for tag in tags_json.get("releases")] except Exception as e: - logging.error("exception, {}".format(e)) + logging.error("exception, %s", e) return [] return release_tags @@ -289,7 +289,7 @@ class RubygemReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -301,7 +301,7 @@ class RubygemReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): if element.get("number"): release_tags.append(element.get("number")) except Exception as e: - logging.error("exception, {}".format(e)) + logging.error("exception, %s", e) return [] return release_tags @@ -323,7 +323,7 @@ class GnuftpReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -356,7 +356,7 @@ class FtpReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -382,7 +382,7 @@ class CmdReleaseTagsMixin(object): sub_proc = subprocess.Popen(cmd_list, stdout=subprocess.PIPE) response = sub_proc.stdout.read().decode("utf-8") if sub_proc.wait(): - logging.warning("{cmd} > encount errors".format(cmd=" ".join(cmd_list))) + logging.warning("%s > encount errors", " ".join(cmd_list)) return response @@ -411,7 +411,7 @@ class SvnReleaseTags(AbsReleaseTags, CmdReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get svn tags".format(repo=url)) + logging.debug("%s : get svn tags", url) if not url: logging.warning("illegal url: \"\"") return [] @@ -465,7 +465,7 @@ class GitReleaseTags(AbsReleaseTags, CmdReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] diff --git a/src/ac/acl/package_yaml/check_yaml.py b/src/ac/acl/package_yaml/check_yaml.py index dad80ac..8ac90cc 100644 --- a/src/ac/acl/package_yaml/check_yaml.py +++ b/src/ac/acl/package_yaml/check_yaml.py @@ -73,7 +73,7 @@ class CheckPackageYaml(BaseCheck): for change_file in diff_files: if change_file == package_yaml: - logger.debug("diff files: {}".format(diff_files)) + logger.debug("diff files: %s", diff_files) return True return False @@ -93,16 +93,16 @@ class CheckPackageYaml(BaseCheck): with open(os.path.join(self._work_dir, yaml_path), 'r') as yaml_data: # load yaml data self._yaml_content = yaml.safe_load(yaml_data) except IOError as e: - logging.warning("package yaml not exist. {}".format(str(e))) + logging.warning("package yaml not exist. %s", str(e)) return WARNING except yaml.YAMLError as exc: - logging.warning("Error parsering YAML: {}".format(str(exc))) + logging.warning("Error parsering YAML: %s", str(exc)) return WARNING result = SUCCESS for keyword in self.PACKAGE_YAML_NEEDED_KEY: if keyword not in self._yaml_content: - logger.error("yaml field {} missing".format(keyword)) + logger.error("yaml field %s missing", keyword) self._is_standard = True result = WARNING return result @@ -129,7 +129,7 @@ class CheckPackageYaml(BaseCheck): tags = release_tags.get_tags(sr) if not tags: - logger.warning("failed to get version by yaml, version_control: {t1}, src_repo: {t2}".format(t1=vc, t2=sr)) + logger.warning("failed to get version by yaml, version_control: %s, src_repo: %s", vc, sr) return WARNING return SUCCESS @@ -154,9 +154,9 @@ class CheckPackageYaml(BaseCheck): if not src_url: src_url = self._spec.get_source("Source") vc = self.VERSION_CTRL_TRANS.get(vc, vc) # 对特殊的版本控制对应的域名进行转换 - logger.debug("version control: {vctrl} source url: {url}".format(vctrl=vc, url=src_url)) + logger.debug("version control: %s source url: %s", vc, src_url) if vc not in src_url: # 通过判断版本控制字段是否在主页url中 判断一致性 - logger.warning("{vc} is not in url: {url}".format(vc=vc, url=src_url)) + logger.warning("%s is not in url: %s", vc, src_url) return WARNING return SUCCESS @@ -196,14 +196,14 @@ class CheckPackageYaml(BaseCheck): src_url = self._spec.get_source("Source0") if not src_url: src_url = self._spec.get_source("Source") - logger.debug("software name: {name} source url: {url}".format(name=software_name, url=src_url)) + logger.debug("software name: %s source url: %s", software_name, src_url) if software_name not in src_url: - logger.warning("{name} is not in source0: {url}".format(name=software_name, url=src_url)) + logger.warning("%s is not in source0: %s", software_name, src_url) return WARNING return SUCCESS def __call__(self, *args, **kwargs): - logger.info("check {} yaml ...".format(self._repo)) + logger.info("check %s yaml ...", self._repo) self._yaml_changed = self.is_change_package_yaml() # yaml文件变更 进行检查 # 因门禁系统限制外网访问权限，将涉及外网访问的检查功能check_repo暂时关闭 return self.start_check_with_order("fields", "repo_domain", "repo_name") diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index 5d5be07..362c1d9 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -49,13 +49,13 @@ class CheckSCA(BaseCheck): with open(self._scanoss_result_output, 'r') as f: result_dirt = json.load(f) except IOError: - logger.error("{} not found, make sure this file exists".format(self._scanoss_result_output)) + logger.error("%s not found, make sure this file exists", self._scanoss_result_output) return FAILED result = result_dirt.get('result') # 保存详细结果到web server - logger.warning("click {} view scanoss detail".format(result_dirt.get('reportUrl'))) + logger.warning("click %s view scanoss detail", result_dirt.get('reportUrl')) return SUCCESS if result else FAILED @@ -66,9 +66,9 @@ class CheckSCA(BaseCheck): :param kwargs: :return: """ - logger.info("check {} sca ...".format(self._repo)) + logger.info("check %s sca ...", self._repo) - logger.debug("args: {}, kwargs: {}".format(args, kwargs)) + logger.debug("args: %s, kwargs: %s", args, kwargs) scanoss_conf = kwargs.get("scanoss", {}) self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index ca0b0ad..540d41f 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -53,7 +53,7 @@ class CheckSpec(BaseCheck): package_yaml = "{}.yaml".format(self._repo) # package yaml file name if len(diff_files) == 1 and diff_files[0] == package_yaml: - logger.debug("diff files: {}".format(diff_files)) + logger.debug("diff files: %s", diff_files) return True return False @@ -94,7 +94,7 @@ class CheckSpec(BaseCheck): # if lts branch, version update is forbidden if self._is_lts_branch(): - logger.debug("lts branch {}".format(self._tbranch)) + logger.debug("lts branch %s", self._tbranch) if RPMSpecAdapter.compare_version(self._spec.version, spec_o.version) == 1: logger.error("version update of lts branch is forbidden") return FAILED @@ -106,8 +106,8 @@ class CheckSpec(BaseCheck): logger.debug("revert commit") return SUCCESS - logger.error("current version: {}-r{}, last version: {}-r{}".format( - self._spec.version, self._spec.release, spec_o.version, spec_o.release)) + logger.error("current version: %s-r%s, last version: %s-r%s", + self._spec.version, self._spec.release, spec_o.version, spec_o.release) return FAILED def check_homepage(self, timeout=30, retrying=3, interval=1): @@ -119,7 +119,7 @@ class CheckSpec(BaseCheck): :return: """ homepage = self._spec.url - logger.debug("homepage: {}".format(homepage)) + logger.debug("homepage: %s", homepage) if not homepage: return SUCCESS @@ -137,15 +137,15 @@ class CheckSpec(BaseCheck): """ patches_spec = set(self._spec.patches) patches_file = set(self._gr.patch_files_not_recursive()) - logger.debug("spec patches: {}".format(patches_spec)) - logger.debug("file patches: {}".format(patches_file)) + logger.debug("spec patches: %s", patches_spec) + logger.debug("file patches: %s", patches_file) result = SUCCESS for patch in patches_spec - patches_file: - logger.error("patch {} lost".format(patch)) + logger.error("patch %s lost", patch) result = FAILED for patch in patches_file - patches_spec: - logger.warning("patch {} redundant".format(patch)) + logger.warning("patch %s redundant", patch) return result @@ -164,7 +164,7 @@ class CheckSpec(BaseCheck): content = "x86-64" if content is not None: - logger.info("exclusive arch \"{}\"".format(content)) + logger.info("exclusive arch \"%s\"", content) try: with open("exclusive_arch", "w") as f: f.write(content) @@ -189,7 +189,7 @@ class CheckSpec(BaseCheck): "last_version": spec.version, "last_release": spec.release, "compare_version": compare_version, "compare_release": compare_release, "compare": compare} - logger.info("{}".format(rs)) + logger.info("%s", rs) try: with open("pkgship_notify", "w") as f: yaml.safe_dump(rs, f) @@ -197,7 +197,7 @@ class CheckSpec(BaseCheck): logger.exception("save pkgship exception") def __call__(self, *args, **kwargs): - logger.info("check {} spec ...".format(self._repo)) + logger.info("check %s spec ...", self._repo) self._ex_exclusive_arch() self._tbranch = kwargs.get("tbranch", None) diff --git a/src/ac/common/gitee_repo.py b/src/ac/common/gitee_repo.py index 996bfe4..c24ca09 100644 --- a/src/ac/common/gitee_repo.py +++ b/src/ac/common/gitee_repo.py @@ -53,16 +53,16 @@ class GiteeRepo(object): for filename in filenames: rel_file_path = os.path.join(dirpath, filename).replace(self._work_dir, "").lstrip("/") if self.is_compress_file(filename): - logger.debug("find compress file: {}".format(rel_file_path)) + logger.debug("find compress file: %s", rel_file_path) self._compress_files.append(rel_file_path) elif self.is_patch_file(filename): - logger.debug("find patch file: {}".format(rel_file_path)) + logger.debug("find patch file: %s", rel_file_path) self._patch_files.append(rel_file_path) elif self.is_spec_file(filename): - logger.debug("find spec file: {}".format(rel_file_path)) + logger.debug("find spec file: %s", rel_file_path) spec_files.append(filename) elif self.is_package_yaml_file(filename): - logger.debug("find yaml file: {}".format(rel_file_path)) + logger.debug("find yaml file: %s", rel_file_path) self.yaml_file = rel_file_path def guess_real_spec_file(): @@ -109,7 +109,7 @@ class GiteeRepo(object): decompress_cmd = "cd {}; timeout 120s tar -C {} -xavf {}".format( self._work_dir, self._decompress_dir, file_path) else: - logger.warning("unsupport compress file: {}".format(file_path)) + logger.warning("unsupport compress file: %s", file_path) return False ret, _, _ = shell_cmd_live(decompress_cmd) @@ -136,20 +136,20 @@ class GiteeRepo(object): :param patch: 补丁 :param max_leading: leading path """ - logger.debug("apply patch {}".format(patch)) + logger.debug("apply patch %s", patch) for patch_dir in [filename for filename in os.listdir(self._decompress_dir) if os.path.isdir(os.path.join(self._decompress_dir, filename))] + ["."]: if patch_dir.startswith(".git"): continue for leading in range(max_leading + 1): - logger.debug("try dir {} -p{}".format(patch_dir, leading)) + logger.debug("try dir %s -p%s", patch_dir, leading) if GitProxy.apply_patch_at_dir(os.path.join(self._decompress_dir, patch_dir), os.path.join(self._work_dir, patch), leading): logger.debug("patch success") self.patch_dir_mapping[patch] = os.path.join(self._decompress_dir, patch_dir) return True - logger.info("apply patch {} failed".format(patch)) + logger.info("apply patch %s failed", patch) return False def apply_all_patches(self, *patches): @@ -166,7 +166,7 @@ class GiteeRepo(object): if patch in set(self._patch_files): rs.append(self.apply_patch(patch)) else: - logger.error("patch {} not exist".format(patch)) + logger.error("patch %s not exist", patch) rs.append(False) return 0 if all(rs) else (1 if any(rs) else -1) diff --git a/src/ac/common/linter.py b/src/ac/common/linter.py index 0b76303..95eacc8 100644 --- a/src/ac/common/linter.py +++ b/src/ac/common/linter.py @@ -73,13 +73,13 @@ class LinterCheck(object): * (E) error, for probable bugs in the code * (F) fatal, if an error occurred which prevented pylint from doing """ - logger.debug("check python file: {}".format(filepath)) + logger.debug("check python file: %s", filepath) # E0401: import module error pylint_cmd = "pylint3 --disable=E0401 {}".format(filepath) ret, out, _ = shell_cmd_live(pylint_cmd, cap_out=True, verbose=True) if ret: - logger.debug("pylint ret, {}".format(ret)) + logger.debug("pylint ret, %s", ret) return cls.get_summary_of_pylint(out) @@ -88,12 +88,12 @@ class LinterCheck(object): """ Check golang code by golint """ - logger.debug("check go file: {}".format(filepath)) + logger.debug("check go file: %s", filepath) golint_cmd = "golint {}".format(filepath) ret, out, _ = shell_cmd_live(golint_cmd, cap_out=True, verbose=True) if ret: - logger.debug("golint error, {}".format(ret)) + logger.debug("golint error, %s", ret) return {} return cls.get_summary_of_golint(out) @@ -103,13 +103,13 @@ class LinterCheck(object): """ Check c/c++ code by splint """ - logger.debug("check c/c++ file: {}".format(filepath)) + logger.debug("check c/c++ file: %s", filepath) splint_cmd = "splint {}".format(filepath) #ret, out, _ = shell_cmd_live(splint_cmd, cap_out=True, verbose=True) ret, out, _ = shell_cmd(splint_cmd) if ret: - logger.debug("splint error, {}".format(ret)) + logger.debug("splint error, %s", ret) return {} return cls.get_summary_of_splint(out) diff --git a/src/ac/common/rpm_spec_adapter.py b/src/ac/common/rpm_spec_adapter.py index a714476..f32ddba 100644 --- a/src/ac/common/rpm_spec_adapter.py +++ b/src/ac/common/rpm_spec_adapter.py @@ -70,7 +70,7 @@ class RPMSpecAdapter(object): """ try: value = self.buildarch - logger.debug("build arch: {}".format(value)) + logger.debug("build arch: %s", value) if "x86_64" in value.lower(): return True @@ -85,7 +85,7 @@ class RPMSpecAdapter(object): """ try: value = self.buildarch - logger.debug("build arch: {}".format(value)) + logger.debug("build arch: %s", value) if "aarch64" in value.lower(): return True @@ -111,7 +111,7 @@ class RPMSpecAdapter(object): version_n = "{}{}".format(version_n, '.0' * (len(version_o.split('.')) - len(version_n.split('.')))) version_o = "{}{}".format(version_o, '.0' * (len(version_n.split('.')) - len(version_o.split('.')))) - logger.debug("compare versions: {} vs {}".format(version_n, version_o)) + logger.debug("compare versions: %s vs %s", version_n, version_o) z = zip(version_n.split("."), version_o.split(".")) for p in z: @@ -121,7 +121,7 @@ class RPMSpecAdapter(object): elif int(p[0]) > int(p[1]): return 1 except ValueError as exc: - logger.debug("check version exception, {}".format(exc)) + logger.debug("check version exception, %s", exc) continue return 0 diff --git a/src/ac/common/scanoss.py b/src/ac/common/scanoss.py index 39ce039..2262702 100644 --- a/src/ac/common/scanoss.py +++ b/src/ac/common/scanoss.py @@ -50,7 +50,7 @@ class ScanOSS(object): try: json_format = json.loads(result) except ValueError: - logger.exception("illegal scanoss result, \"{}\"".format(result)) + logger.exception("illegal scanoss result, \"%s\"", result) return True snippets = 0 @@ -67,7 +67,7 @@ class ScanOSS(object): detail_trs.append(self.__class__.detail_trs(filename, item)) - logger.debug("snippets: {}, files: {}".format(snippets, files)) + logger.debug("snippets: %s, files: %s", snippets, files) detail = "{th}{trs}
".format( th=self.__class__.detail_th(), trs="\n".join(detail_trs)) @@ -161,7 +161,7 @@ class ScanOSS(object): :param directory: 需要扫描的目录 :return: """ - logger.debug("scan dir: {}".format(directory)) + logger.debug("scan dir: %s", directory) #scanoss_cmd = "scanner.py --format {} {} --apiurl {} {}".format( # "plain", "--key {}".format(self._key) if self._key else "", self._api_url, directory) scanoss_cmd = "scanner.py --blacklist {} --format {} {} --apiurl {} {}".format( @@ -169,8 +169,8 @@ class ScanOSS(object): ret, out, err = shell_cmd(scanoss_cmd) if ret: - logger.error("scanoss error, {}".format(ret)) - logger.error("{}".format(err)) + logger.error("scanoss error, %s", ret) + logger.error("%s", err) return True return self.result_analysis(out) diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index a205dc9..f208b00 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -49,7 +49,7 @@ class AC(object): self.load_check_elements_from_acl_directory(acl_path) self.load_check_elements_from_conf(conf, community) - logger.debug("check list: {}".format(self._ac_check_elements)) + logger.debug("check list: %s", self._ac_check_elements) @staticmethod def is_repo_support_check(repo, check_element): @@ -74,7 +74,7 @@ class AC(object): """ for element in self._ac_check_elements: check_element = self._ac_check_elements[element] - logger.debug("check {}".format(element)) + logger.debug("check %s", element) # show in gitee, must starts with "check_" hint = check_element.get("hint", "check_{}".format(element)) @@ -82,25 +82,25 @@ class AC(object): hint = "check_{}".format(hint) if not self.__class__.is_repo_support_check(repo, check_element): - logger.debug("{} not support check".format(repo)) + logger.debug("%s not support check", repo) continue # import module module_path = check_element.get("module", "{}.check_{}".format(element, element)) # eg: spec.check_spec try: module = importlib.import_module("." + module_path, self._acl_package) - logger.debug("load module {} succeed".format(module_path)) + logger.debug("load module %s succeed", module_path) except ImportError as exc: - logger.exception("import module {} exception, {}".format(module_path, exc)) + logger.exception("import module %s exception, %s", module_path, exc) continue # import entry entry_name = check_element.get("entry", "Check{}".format(element.capitalize())) try: entry = getattr(module, entry_name) - logger.debug("load entry \"{}\" succeed".format(entry_name)) + logger.debug("load entry \"%s\" succeed", entry_name) except AttributeError as exc: - logger.warning("entry \"{}\" not exist in module {}, {}".format(entry_name, module_path, exc)) + logger.warning("entry \"%s\" not exist in module %s, %s", entry_name, module_path, exc) continue # new a instance @@ -109,26 +109,26 @@ class AC(object): entry = entry(workspace, repo, check_element) # new a instance except Exception as exc: self._ac_check_result.append({"name": hint, "result": FAILED.val}) - logger.exception("new a instance of class {} exception, {}".format(entry_name, exc)) + logger.exception("new a instance of class %s exception, %s", entry_name, exc) continue if not callable(entry): # check callable - logger.warning("entry {} not callable".format(entry_name)) + logger.warning("entry %s not callable", entry_name) continue # do ac check try: result = entry(**kwargs) - logger.debug("check result {} {}".format(element, result)) + logger.debug("check result %s %s", element, result) except Exception as exc: - logger.exception("check exception, {} {}".format(element, exc)) + logger.exception("check exception, %s %s", element, exc) continue self._ac_check_result.append({"name": hint, "result": result.val}) dataset.set_attr("access_control.build.acl.{}".format(element), result.hint) dataset.set_attr("access_control.build.content", self._ac_check_result) - logger.debug("ac result: {}".format(self._ac_check_result)) + logger.debug("ac result: %s", self._ac_check_result) def load_check_elements_from_acl_directory(self, acl_dir): """ @@ -150,18 +150,18 @@ class AC(object): with open(conf_file, "r") as f: content = yaml.safe_load(f) except IOError: - logger.exception("ac conf file {} not exist".format(conf_file)) + logger.exception("ac conf file %s not exist", conf_file) return except YAMLError: logger.exception("illegal conf file format") return elements = content.get(community, {}) - logger.debug("community \"{}\" conf: {}".format(community, elements)) + logger.debug("community \"%s\" conf: %s", community, elements) for name in elements: if name in self._ac_check_elements: if elements[name].get("exclude"): - logger.debug("exclude: {}".format(name)) + logger.debug("exclude: %s", name) self._ac_check_elements.pop(name) else: self._ac_check_elements[name] = elements[name] @@ -172,7 +172,7 @@ class AC(object): :param ac_file: :return: """ - logger.debug("save ac result to file {}".format(ac_file)) + logger.debug("save ac result to file %s", ac_file) with open(ac_file, "w") as f: f.write("ACL={}".format(json.dumps(self._ac_check_result))) @@ -222,10 +222,10 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("ac") - logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.community, args.repo)) + logger.info("using credential %s", args.account.split(":")[0]) + logger.info("cloning repository https://gitee.com/%s/%s.git ", args.community, args.repo) logger.info("clone depth 4") - logger.info("checking out pull request {}".format(args.pr)) + logger.info("checking out pull request %s", args.pr) # notify gitee from src.proxy.gitee_proxy import GiteeProxy diff --git a/src/ac/framework/ac_base.py b/src/ac/framework/ac_base.py index 7deb3b5..9f3e9d7 100644 --- a/src/ac/framework/ac_base.py +++ b/src/ac/framework/ac_base.py @@ -57,27 +57,27 @@ class BaseCheck(object): result = SUCCESS for name in items: try: - logger.debug("check {}".format(name)) + logger.debug("check %s", name) method = getattr(self, "check_{}".format(name)) rs = method() - logger.debug("{} -> {}".format(name, rs)) + logger.debug("%s -> %s", name, rs) except Exception as e: # 忽略代码错误 - logger.exception("internal error: {}".format(e)) + logger.exception("internal error: %s", e) continue ignored = True if self._conf and name in self._conf.get("ignored", []) else False - logger.debug("{} ignore: {}".format(name, ignored)) + logger.debug("%s ignore: %s", name, ignored) if rs is SUCCESS: - logger.info("check {:<30}pass".format(name)) + logger.info("check %s pass", name) elif rs is WARNING: - logger.warning("check {:<30}warning{}".format(name, " [ignored]" if ignored else "")) + logger.warning("check %s warning %s", name, " [ignored]" if ignored else "") elif rs is FAILED: - logger.error("check {:<30}fail{}".format(name, " [ignored]" if ignored else "")) + logger.error("check %s fail %s", name, " [ignored]" if ignored else "") else: # never here - logger.exception("check {:<30}exception{}".format(name, " [ignored]" if ignored else "")) + logger.exception("check %s exception %s", name, " [ignored]" if ignored else "") continue if not ignored: @@ -91,6 +91,6 @@ class BaseCheck(object): """ members = inspect.getmembers(self, inspect.ismethod) items = [member[0].replace("check_", "") for member in members if member[0].startswith("check_")] - logger.debug("check items: {}".format(items)) + logger.debug("check items: %s", items) return self.start_check_with_order(*items) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index fbefcc8..149b701 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -49,7 +49,7 @@ class ExtraWork(object): try: with open(pkgship_meta_path, "r") as f: pkgship_meta = yaml.safe_load(f) - logger.debug("pkgship meta: {}".format(pkgship_meta)) + logger.debug("pkgship meta: %s", pkgship_meta) if pkgship_meta.get("compare_version") == 1: # version upgrade logger.debug("pkgship: notify") return True @@ -93,8 +93,8 @@ class ExtraWork(object): #get rpms curr_rpm = self._rpm_package.main_package_local() last_rpm = self._rpm_package.last_main_package(package_arch, package_url) - logger.debug("curr_rpm: {}".format(curr_rpm)) - logger.debug("last_rpm: {}".format(last_rpm)) + logger.debug("curr_rpm: %s", curr_rpm) + logger.debug("last_rpm: %s", last_rpm) if not curr_rpm or not last_rpm: logger.info("no rpms") return @@ -108,8 +108,8 @@ class ExtraWork(object): debuginfos = None curr_rpm_debug = self._rpm_package.debuginfo_package_local() last_rpm_debug = self._rpm_package.last_debuginfo_package(package_arch, package_url) - logger.debug("curr_rpm_debug: {}".format(curr_rpm_debug)) - logger.debug("last_rpm_debug: {}".format(last_rpm_debug)) + logger.debug("curr_rpm_debug: %s", curr_rpm_debug) + logger.debug("last_rpm_debug: %s", last_rpm_debug) if curr_rpm_debug and last_rpm_debug: debuginfos = [last_rpm_debug, curr_rpm_debug] @@ -123,9 +123,9 @@ class ExtraWork(object): check_abi = CheckAbi(result_output_file=output, input_rpms_path=related_rpms_url) ret = check_abi.process_with_rpm(rpms, debuginfos) if ret == 1: - logger.error("check abi error: {}".format(ret)) + logger.error("check abi error: %s", ret) else: - logger.debug("check abi ok: {}".format(ret)) + logger.debug("check abi ok: %s", ret) if os.path.exists(output): # change of abi @@ -135,7 +135,7 @@ class ExtraWork(object): else: comment = {"name": "check_abi/{}/{}".format(package_arch, self._repo), "result": "SUCCESS"} - logger.debug("check abi comment: {}".format(comment)) + logger.debug("check abi comment: %s", comment) try: with open(comment_file, "r") as f: # one repo with multi build package comments = yaml.safe_load(f) @@ -151,7 +151,7 @@ class ExtraWork(object): logger.exception("yaml load check abi comment file exception") comments.append(comment) - logger.debug("check abi comments: {}".format(comments)) + logger.debug("check abi comments: %s", comments) try: with open(comment_file, "w") as f: yaml.safe_dump(comments, f) # list @@ -170,14 +170,14 @@ class ExtraWork(object): # 1. prepare install root directory _ = not os.path.exists(install_root) and os.mkdir(install_root) - logger.info("create install root directory: {}".format(install_root)) + logger.info("create install root directory: %s", install_root) repo_name_prefix = "check_install" # 2. prepare repo repo_source = OBSRepoSource("http://119.3.219.20:82") # obs 实时构建repo地址 repo_config = repo_source.generate_repo_info(branch_name, arch, "check_install") - logger.info("repo source config:\n{}".format(repo_config)) + logger.info("repo source config:\n%s", repo_config) # write to /etc/yum.repos.d with open("obs_realtime.repo", "w+") as f: @@ -189,18 +189,18 @@ class ExtraWork(object): for name, package in self._rpm_package.iter_all_rpm(): # ignore debuginfo rpm if "debuginfo" in name or "debugsource" in name: - logger.debug("ignore debug rpm: {}".format(name)) + logger.debug("ignore debug rpm: %s", name) continue names.append(name) packages.append(package) - logger.info("install rpms: {}".format(names)) + logger.info("install rpms: %s", names) if packages: check_install_cmd = "sudo dnf install -y --installroot={} --setopt=reposdir=. {}".format( install_root, " ".join(packages)) ret, _, err = shell_cmd_live(check_install_cmd, verbose=True) if ret: - logger.error("install rpms error, {}, {}".format(ret, err)) + logger.error("install rpms error, %s, %s", ret, err) else: logger.info("install rpm success") diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 442ebbb..5d778bc 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -82,7 +82,7 @@ class Comment(object): base_job_name = os.environ.get("JOB_NAME") base_build_id = os.environ.get("BUILD_ID") base_build_id = int(base_build_id) - logger.debug("base_job_name: {}, base_build_id: {}".format(base_job_name, base_build_id)) + logger.debug("base_job_name: %s, base_build_id: %s", base_job_name, base_build_id) base_build = jenkins_proxy.get_build(base_job_name, base_build_id) logger.debug("get base build") self._up_builds = jenkins_proxy.get_upstream_builds(base_build) @@ -120,7 +120,7 @@ class Comment(object): try: acl = json.loads(os.environ["ACL"]) - logger.debug("ac result: {}".format(acl)) + logger.debug("ac result: %s", acl) except ValueError: logger.exception("invalid ac result format") return [] @@ -138,7 +138,7 @@ class Comment(object): comments.append(self.__class__.comment_html_table_tr_rowspan( item["name"], ac_result.emoji, ac_result.hint)) - logger.info("ac comment: {}".format(comments)) + logger.info("ac comment: %s", comments) return comments @@ -158,7 +158,7 @@ class Comment(object): comments.append(self.__class__.comment_html_table_tr( name, ac_result.emoji, ac_result.hint, "{}{}".format(build_url, "console"), build.buildno)) - logger.info("build comment: {}".format(comments)) + logger.info("build comment: %s", comments) return comments @@ -178,25 +178,25 @@ class Comment(object): return False for check_abi_comment_file in self._check_abi_comment_files: - logger.debug("check abi comment file: {}".format(check_abi_comment_file)) + logger.debug("check abi comment file: %s", check_abi_comment_file) if not os.path.exists(check_abi_comment_file): # check abi评论文件存在 continue for build in builds: name = build.job._data["fullName"] - logger.debug("check build {}".format(name)) + logger.debug("check build %s", name) if not match(name, check_abi_comment_file): # 找到匹配的jenkins build continue - logger.debug("build \"{}\" match".format(name)) + logger.debug("build \"%s\" match", name) status = build.get_status() - logger.debug("build state: {}".format(status)) + logger.debug("build state: %s", status) if ACResult.get_instance(status) == SUCCESS: # 保证build状态成功 with open(check_abi_comment_file, "r") as f: try: content = yaml.safe_load(f) except YAMLError: # yaml base exception logger.exception("illegal yaml format of check abi comment file ") - logger.debug("comment: {}".format(content)) + logger.debug("comment: %s", content) for item in content: ac_result = ACResult.get_instance(item.get("result")) comments.append(self.__class__.comment_html_table_tr( @@ -204,7 +204,7 @@ class Comment(object): "markdown" if "link" in item else "", hashtag=False)) break - logger.info("check abi comment: {}".format(comments)) + logger.info("check abi comment: %s", comments) return comments diff --git a/src/build/obs_repo_source.py b/src/build/obs_repo_source.py index 3548350..638e44c 100644 --- a/src/build/obs_repo_source.py +++ b/src/build/obs_repo_source.py @@ -69,35 +69,35 @@ class OBSRepoSource(object): # openstack common url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part_common, arch) if do_requests("GET", url) == 0: - logger.debug("add openstack common repo: {}".format(url)) + logger.debug("add openstack common repo: %s", url) repo_config += self.repo_format("openstack_common", repo_name_prefix + "_openstack_common", url) # openstack base url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part_base, arch) if do_requests("GET", url) == 0: - logger.debug("add openstack base repo: {}".format(url)) + logger.debug("add openstack base repo: %s", url) repo_config += self.repo_format("openstack_base", repo_name_prefix + "_openstack_base", url) else: obs_path_part = branch.replace("-", ":/") - logger.debug("branch={}, obs_path_part={}".format(branch, obs_path_part)) + logger.debug("branch=%s, obs_path_part=%s", branch, obs_path_part) # main url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part, arch) if do_requests("GET", url) == 0: - logger.debug("add main repo: {}".format(url)) + logger.debug("add main repo: %s", url) repo_config += self.repo_format(repo_name_prefix + "_main", repo_name_prefix + "_main", url) # epol url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part + ":/Epol", arch) if do_requests("GET", url) == 0: - logger.debug("add epol repo: {}".format(url)) + logger.debug("add epol repo: %s", url) repo_config += self.repo_format(repo_name_prefix + "_epol", repo_name_prefix + "_epol", url) # extras url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part + ":/Extras", arch) if do_requests("GET", url) == 0: - logger.debug("add extras repo: {}".format(url)) + logger.debug("add extras repo: %s", url) repo_config += self.repo_format(repo_name_prefix + "_extras", repo_name_prefix + "_extras", url)\ return repo_config diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index dd9b32e..be38c69 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -102,16 +102,16 @@ class SinglePackageBuild(object): # osc build for repo in repos: if repo["state"] == "excluded" and repo["mpac"] == "raspberrypi-kernel": - logger.info("repo {}:{} excluded".format(repo["repo"], repo["mpac"])) + logger.info("repo %s:%s excluded", repo["repo"], repo["mpac"]) continue root_build = repo["mpac"] in self.PACKAGES_USE_ROOT if not OBSProxy.build_package( project, self._package, repo["repo"], self._arch, spec, repo["mpac"], root_build=root_build, disable_cpio=True): - logger.error("build {} ... failed".format(repo["repo"])) + logger.error("build %s ... failed", repo["repo"]) return 3 - logger.info("build {} ... ok".format(repo["repo"])) + logger.info("build %s ... ok", repo["repo"]) logger.debug("build all repos ... finished") @@ -189,7 +189,7 @@ class SinglePackageBuild(object): ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: - logger.error("prepare build environ error, {}".format(ret)) + logger.error("prepare build environ error, %s", ret) return False return True @@ -203,31 +203,31 @@ class SinglePackageBuild(object): :return: """ if self._branch in self.BUILD_IGNORED_GITEE_BRANCH: - logger.error("branch \"{}\" ignored".format(self._branch)) + logger.error("branch \"%s\" ignored", self._branch) return 0 if self._branch not in self.GITEE_BRANCH_PROJECT_MAPPING: - logger.error("branch \"{}\" not support yet".format(self._branch)) + logger.error("branch \"%s\" not support yet", self._branch) return 1 has_any_repo_build = False for project in self.GITEE_BRANCH_PROJECT_MAPPING.get(self._branch): - logger.debug("start build project {}".format(project)) + logger.debug("start build project %s", project) obs_repos = self.get_need_build_obs_repos(project) if not obs_repos: - logger.info("all repos ignored of project {}".format(project)) + logger.info("all repos ignored of project %s", project) continue - logger.debug("build obs repos: {}".format(obs_repos)) + logger.debug("build obs repos: %s", obs_repos) has_any_repo_build = True ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) if ret > 0: - logger.debug("build run return {}".format(ret)) - logger.error("build {} {} {} ... {}".format(project, self._package, self._arch, "failed")) + logger.debug("build run return %s", ret) + logger.error("build %s %s %s ... %s", project, self._package, self._arch, "failed") return 1 # finish if any error else: - logger.info("build {} {} {} ... {}".format(project, self._package, self._arch, "ok")) + logger.info("build %s %s %s ... %s", project, self._package, self._arch, "ok") # if no repo build, regard as fail if not has_any_repo_build: @@ -269,10 +269,10 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") - logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git ".format(args.owner, args.repo)) + logger.info("using credential %s", args.account.split(":")[0]) + logger.info("cloning repository https://gitee.com/%s/%s.git ", args.owner, args.repo) logger.info("clone depth 1") - logger.info("checking out pull request {}".format(args.pr)) + logger.info("checking out pull request %s", args.pr) from src.utils.dist_dataset import DistDataset from src.proxy.git_proxy import GitProxy diff --git a/src/jobs/jenkins_job.py b/src/jobs/jenkins_job.py index 3978493..811058c 100755 --- a/src/jobs/jenkins_job.py +++ b/src/jobs/jenkins_job.py @@ -59,7 +59,7 @@ class JenkinsJobs(object): :param interval: 每次batch请求后sleep时间（秒）， :return: """ - logger.info("{} jobs {}".format(action, jobs)) + logger.info("%s jobs %s", action, jobs) real_jobs = self.get_real_target_jobs(jobs, exclude_jobs if exclude_jobs else []) def run_once(target_jobs): @@ -71,14 +71,14 @@ class JenkinsJobs(object): for index in range(batch): works = [gevent.spawn(self.dispatch, action, job, jenkins_proxy) for job in target_jobs[index * concurrency: (index + 1) * concurrency]] - logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) + logger.info("%s works, %s/%s ", len(works), index + 1, batch) gevent.joinall(works) for work in works: if work.value["result"]: - logger.info("{} job {} ... ok".format(action, work.value["job"])) + logger.info("%s job %s ... ok", action, work.value["job"]) else: _failed_jobs.append(work.value["job"]) - logger.error("{} job {} ... failed".format(action, work.value["job"])) + logger.error("%s job %s ... failed", action, work.value["job"]) time.sleep(interval) @@ -89,12 +89,12 @@ class JenkinsJobs(object): for index in range(retry): if not failed_jobs: break - logger.info("{} jobs failed, retrying {}/{}".format(len(failed_jobs), index + 1, retry)) + logger.info("%s jobs failed, retrying %s/%s", len(failed_jobs), index + 1, retry) failed_jobs = run_once(failed_jobs) if failed_jobs: - logger.warning("{} failed jobs".format(len(failed_jobs))) - logger.warning("{}{}".format(",".join(failed_jobs[:100]), "..." if len(failed_jobs) > 100 else "")) + logger.warning("%s failed jobs", len(failed_jobs)) + logger.warning("%s%s", ",".join(failed_jobs[:100]), "..." if len(failed_jobs) > 100 else "") def dispatch(self, action, job, jenkins_proxy): """ @@ -148,7 +148,7 @@ class SrcOpenEulerJenkinsJobs(JenkinsJobs): with open(os.path.join(exclusive_arch_path, filename), "r") as f: arches = f.readline() self._exclusive_arch[filename] = [arch.strip() for arch in arches.split(",")] - logger.debug("exclusive arch: {}".format(self._exclusive_arch)) + logger.debug("exclusive arch: %s", self._exclusive_arch) def get_real_target_jobs(self, jobs, exclude_jobs): """ @@ -274,7 +274,7 @@ class OpenEulerJenkinsJobs(SrcOpenEulerJenkinsJobs): # build script = self.guess_build_script(buddy["repo"]) - logger.debug("guess build script: {}".format("script")) + logger.debug("guess build script: %s", "script") ele = root.findall("buiders/hudson.task.Shell/command") if ele: # replace first command diff --git a/src/jobs/obs_meta_strategy.py b/src/jobs/obs_meta_strategy.py index 6b30620..24b0c40 100755 --- a/src/jobs/obs_meta_strategy.py +++ b/src/jobs/obs_meta_strategy.py @@ -79,20 +79,20 @@ class ObsMetaStrategy(object): if filename == "_service": _service = os.path.join(dirpath, filename) try: - logger.debug("analysis {}".format(_service)) + logger.debug("analysis %s", _service) tree = ET.parse(_service) elements = tree.findall(".//param[@name=\"url\"]") # next/openEuler/zip except: - logger.exception("invalid xml format, {}".format(_service)) + logger.exception("invalid xml format, %s", _service) continue _repos = [element.text.strip("/").split("/")[-1] for element in elements] # eg: next/openEuler/zip - logger.debug("get repos: {}".format(_repos)) + logger.debug("get repos: %s", _repos) if any([repo in repos for repo in _repos]): package = dirpath.strip("/").split("/")[-1] # eg: master/openEuler:Mainline/zip/_services index += 1 - logger.info("{} {}...ok".format(index, _service)) - logger.info("package: {}, repos: {}".format(package, _repos)) + logger.info("%s %s...ok", index, _service) + logger.info("package: %s, repos: %s", package, _repos) for repo in _repos: self._package_repo[package].add(repo) self._repo_package[repo].add(package) diff --git a/src/jobs/repo_mapping.py b/src/jobs/repo_mapping.py index 49b1765..318f2b2 100755 --- a/src/jobs/repo_mapping.py +++ b/src/jobs/repo_mapping.py @@ -38,9 +38,9 @@ class RepoMapping(object): self._exclude_repos = kwargs.get("exclude_jobs") if kwargs.get("exclude_jobs") else [] self._repo_mapping = {} # 保存结果 self._ignored_repos = self._load_ignore_repo(ignored_repos_path, ignored_repos_key) - logger.debug("ignored repos: {}".format(self._ignored_repos)) + logger.debug("ignored repos: %s", self._ignored_repos) self._community_repos = self._load_community_repo(community_path) # 社区repos - logger.debug("community repos: {}".format(self._community_repos)) + logger.debug("community repos: %s", self._community_repos) @staticmethod def _load_ignore_repo(conf_file, ignored_repos_key): @@ -55,7 +55,7 @@ class RepoMapping(object): handler = yaml.safe_load(f) return handler.get(ignored_repos_key, []) except IOError as e: - logger.warning("{} not exist".format(conf_file)) + logger.warning("%s not exist", conf_file) return [] @staticmethod @@ -69,10 +69,10 @@ class RepoMapping(object): with open(community_path, "r") as f: handler = yaml.safe_load(f) repos = {item["name"]: item["type"] for item in handler["repositories"]} - logger.info("repos from community: {}".format(len(repos))) + logger.info("repos from community: %s", len(repos)) return repos except IOError as e: - logger.warning("{} not exist".format(community_path)) + logger.warning("%s not exist", community_path) return [] def _is_valid_repo(self, repo): diff --git a/src/proxy/es_proxy.py b/src/proxy/es_proxy.py index b439951..3b18141 100644 --- a/src/proxy/es_proxy.py +++ b/src/proxy/es_proxy.py @@ -46,10 +46,10 @@ class ESProxy(object): :return: """ try: - logger.debug("es insert: {}".format(body)) + logger.debug("es insert: %s", body) rs = self._es.index(index, body=body) - logger.debug("insert result: {}".format(rs)) + logger.debug("insert result: %s", rs) return rs["result"] == "created" except elasticsearch.ElasticsearchException: logger.exception("elastic search insert document exception") @@ -62,9 +62,9 @@ class ESProxy(object): :param body: :return: """ - logger.debug("es search: {}".format(body)) + logger.debug("es search: %s", body) rs = self._es.search(index=index, body=body) - logger.debug("result: {}".format(rs)) + logger.debug("result: %s", rs) return rs['hits']['hits'] @@ -89,9 +89,9 @@ class ESProxy(object): """ try: body = {"query": query, "script": script} - logger.debug("es update: {}".format(body)) + logger.debug("es update: %s", body) rs = self._es.update_by_query(index, body=body) - logger.debug("update result: {}".format(rs)) + logger.debug("update result: %s", rs) return True except elasticsearch.ElasticsearchException: diff --git a/src/proxy/git_proxy.py b/src/proxy/git_proxy.py index 87fe232..a7bca0b 100644 --- a/src/proxy/git_proxy.py +++ b/src/proxy/git_proxy.py @@ -50,7 +50,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(init_cmd) if ret: - logger.warning("init repository failed, {}".format(ret)) + logger.warning("init repository failed, %s", ret) return None return cls(repo_dir) @@ -70,8 +70,8 @@ class GitProxy(object): self._repo_dir, "--progress" if progress else "", depth, url, pull_request, pull_request) ret, out, _ = shell_cmd_live(fetch_cmd, cap_out=True, cmd_verbose=False) if ret: - logger.error("git fetch failed, {}".format(ret)) - logger.error("{}".format(out)) + logger.error("git fetch failed, %s", ret) + logger.error("%s", out) return False return True @@ -86,7 +86,7 @@ class GitProxy(object): get_content_cmd = "cd {}; git show {}:{}".format(self._repo_dir, commit, file_path) ret, out, _ = shell_cmd_live(get_content_cmd, cap_out=True) if ret: - logger.warning("get file content of commit failed, {}".format(ret)) + logger.warning("get file content of commit failed, %s", ret) return None f = StringIO() @@ -106,7 +106,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(diff_files_cmd, cap_out=True) if ret: - logger.error("get diff files of commits failed, {}".format(ret)) + logger.error("get diff files of commits failed, %s", ret) return [] return out @@ -121,7 +121,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(extract_file_cmd, cap_out=True) if ret: - logger.error("extract diff files of patch failed, {}".format(ret)) + logger.error("extract diff files of patch failed, %s", ret) return [] return [line.split()[-1] for line in out] @@ -137,7 +137,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(apply_patch_cmd) if ret: - #logger.error("apply patch failed, {}".format(ret)) + #logger.error("apply patch failed, %s", ret) return False return True @@ -156,7 +156,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(apply_patch_cmd) if ret: - #logger.error("apply patch failed, {}".format(ret)) + #logger.error("apply patch failed, %s", ret) return False return True @@ -171,7 +171,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(get_commit_cmd, cap_out=True) if ret: - logger.error("get commit id of index failed, {}".format(ret)) + logger.error("get commit id of index failed, %s", ret) return None return out[0] @@ -186,7 +186,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(checkout_cmd) if ret: - logger.warning("checkout failed, {}".format(ret)) + logger.warning("checkout failed, %s", ret) return False return True @@ -201,7 +201,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(checkout_cmd) if ret: - logger.warning("checkout failed, {}".format(ret)) + logger.warning("checkout failed, %s", ret) return False return True @@ -224,7 +224,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(tree_hashes_cmd, cap_out=True) if ret: - logger.error("get tree hashes failed, {}".format(ret)) + logger.error("get tree hashes failed, %s", ret) return None return out @@ -243,7 +243,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(fetch_cmd) if ret: - logger.error("fetch failed, {}".format(ret)) + logger.error("fetch failed, %s", ret) return False return True diff --git a/src/proxy/gitee_proxy.py b/src/proxy/gitee_proxy.py index 3e30c60..3cfa96f 100644 --- a/src/proxy/gitee_proxy.py +++ b/src/proxy/gitee_proxy.py @@ -35,7 +35,7 @@ class GiteeProxy(object): :param comment: 评论内容 :return: 0成功，其它失败 """ - logger.debug("comment pull request {}".format(pr)) + logger.debug("comment pull request %s", pr) comment_pr_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/comments".format(self._owner, self._repo, pr) data = {"access_token": self._token, "body": comment} @@ -58,7 +58,7 @@ class GiteeProxy(object): logger.debug("create tags, but no tags") return True - logger.debug("create tags {} of pull request {}".format(tags, pr)) + logger.debug("create tags %s of pull request %s", tags, pr) pr_tag_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/labels?access_token={}".format( self._owner, self._repo, pr, self._token) @@ -81,7 +81,7 @@ class GiteeProxy(object): logger.debug("replace tags, but no tags") return True - logger.debug("replace all tags with {} of pull request {}".format(tags, pr)) + logger.debug("replace all tags with %s of pull request %s", tags, pr) pr_tag_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/labels?access_token={}".format( self._owner, self._repo, pr, self._token) @@ -99,7 +99,7 @@ class GiteeProxy(object): :param tag: 标签 :return: 0成功，其它失败 """ - logger.debug("delete tag {} of pull request {}".format(tag, pr)) + logger.debug("delete tag %s of pull request %s", tag, pr) pr_tag_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/labels/{}?access_token={}".format( self._owner, self._repo, pr, tag, self._token) @@ -128,7 +128,7 @@ class GiteeProxy(object): """ handler = yaml.safe_load(response.text) repos.update({item["name"]: item["type"] for item in handler["repositories"]}) - logger.info("repos from community: {}".format(len(repos))) + logger.info("repos from community: %s", len(repos)) community_repo_url = "https://gitee.com/openeuler/community/raw/master/repository/src-openeuler.yaml" logger.info("requests repos from community, this will take multi seconds") @@ -143,7 +143,7 @@ class GiteeProxy(object): :param state: pr状态 :return: str or None """ - logger.debug("get last pull request committer, branch: {}, state: {}".format(branch, state)) + logger.debug("get last pull request committer, branch: %s, state: %s", branch, state) pr_url = "https://gitee.com/api/v5/repos/{}/{}/pulls?access_token={}&state={}&base={}" \ "&page=1&per_page=1".format(self._owner, self._repo, self._token, state, branch) @@ -161,7 +161,7 @@ class GiteeProxy(object): if handler: try: committer[0] = handler[0]["user"]["login"] - logger.debug("get last pr committer: {}".format(committer)) + logger.debug("get last pr committer: %s", committer) except KeyError: logger.exception("extract committer info from gitee exception") diff --git a/src/proxy/jenkins_proxy.py b/src/proxy/jenkins_proxy.py index b4b035b..ea40dfc 100644 --- a/src/proxy/jenkins_proxy.py +++ b/src/proxy/jenkins_proxy.py @@ -55,7 +55,7 @@ class JenkinsProxy(object): self._jenkins.create_job(job, config) return True except Exception as e: - logger.exception("create job exception, {}".format(e)) + logger.exception("create job exception, %s", e) return False def update_job(self, job, config): @@ -70,7 +70,7 @@ class JenkinsProxy(object): jks_job.update_config(config) return True except Exception as e: - logger.exception("update job exception, {}".format(e)) + logger.exception("update job exception, %s", e) return False def get_config(self, job): @@ -82,7 +82,7 @@ class JenkinsProxy(object): try: return self._jenkins[job].get_config() except Exception as e: - logger.exception("get config exception, {}".format(e)) + logger.exception("get config exception, %s", e) return None def get_build(self, job, build_no): @@ -95,7 +95,7 @@ class JenkinsProxy(object): try: return self._jenkins[job].get_build(build_no) except Exception as e: - logger.exception("get job build exception, {}".format(e)) + logger.exception("get job build exception, %s", e) return None @classmethod @@ -109,7 +109,7 @@ class JenkinsProxy(object): parent_build = build.get_upstream_build() return parent_build.get_upstream_build() if parent_build else None except Exception as e: - logger.exception("get grandpa build exception, {}".format(e)) + logger.exception("get grandpa build exception, %s", e) return None def _get_upstream_jobs(self, job): @@ -119,22 +119,22 @@ class JenkinsProxy(object): :param job: Jenkins Job object :return: """ - logger.debug("get upstream jobs of {}".format(job._data["fullName"])) + logger.debug("get upstream jobs of %s", job._data["fullName"]) jobs = [] for project in job._data["upstreamProjects"]: # but is the only way of get upstream projects info url = project.get("url") name = project.get("name") - logger.debug("upstream project: {} {}".format(url, name)) + logger.debug("upstream project: %s %s", url, name) m = re.match("(.*)/job/.*", url) # remove last part of job url, greedy match base_url = m.group(1) - logger.debug("base url {}".format(base_url)) + logger.debug("base url %s", base_url) try: j = jenkins.Jenkins(base_url, self._username, self._token, timeout=self._timeout) jobs.append(j[name]) except Exception as e: - logger.exception("get job of {} exception".format(url)) + logger.exception("get job of %s exception", url) continue return jobs @@ -167,17 +167,17 @@ class JenkinsProxy(object): cause_build = cause_job.get_build(cause_build_id) cause_cause_build_id = cause_build.get_upstream_build_number() - logger.debug("cause_build_id: {}, cause_job_name: {}, cause_cause_build_id: {}".format( - cause_build_id, cause_job_name, cause_cause_build_id)) + logger.debug("cause_build_id: %s, cause_job_name: %s, cause_cause_build_id: %s", + cause_build_id, cause_job_name, cause_cause_build_id) upstream_builds = [] for upstream_job in upstream_jobs: - logger.debug("{}".format(upstream_job._data["fullName"])) + logger.debug("%s", upstream_job._data["fullName"]) for build_id in upstream_job.get_build_ids(): - logger.debug("try build id {}".format(build_id)) + logger.debug("try build id %s", build_id) a_build = upstream_job.get_build(build_id) if a_build.get_upstream_build_number() == cause_cause_build_id: - logger.debug("build id {} match".format(build_id)) + logger.debug("build id %s match", build_id) upstream_builds.append(a_build) break diff --git a/src/proxy/kafka_proxy.py b/src/proxy/kafka_proxy.py index c7dcde4..110b46e 100644 --- a/src/proxy/kafka_proxy.py +++ b/src/proxy/kafka_proxy.py @@ -49,12 +49,12 @@ class KafkaProducerProxy(object): :return: """ try: - logger.debug("kafka send: {}, {}".format(key, value)) + logger.debug("kafka send: %s, %s", key, value) future = self._kp.send(topic, value=value, key=key) rs = future.get(timeout=self._timeout) - logger.debug("kafka send result: {}".format(rs)) + logger.debug("kafka send result: %s", rs) return True except errors.KafkaTimeoutError: logger.exception("kafka send timeout exception") diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index 9ccabc5..98fd978 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -45,7 +45,7 @@ class OBSProxy(object): cmd = "osc ll {} {}".format(project, package) ret, rs, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.error("list project package error, {}".format(ret)) + logger.error("list project package error, %s", ret) return None return rs @@ -62,7 +62,7 @@ class OBSProxy(object): "--show-exclude" if show_exclude else "", project, package, arch) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.debug("list obs repos of arch error, {}".format(ret)) + logger.debug("list obs repos of arch error, %s", ret) return [] rs = [] @@ -88,7 +88,7 @@ class OBSProxy(object): cmd = "osc results {} --csv |grep {} | awk -F';' '{{print $1}}'".format(project, state) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.debug("list package of state error, {}".format(ret)) + logger.debug("list package of state error, %s", ret) return [] return out @@ -105,11 +105,11 @@ class OBSProxy(object): _ = os.path.isdir(project) and shutil.rmtree(project) cmd = "osc co {} {}".format(project, package) - logger.info("osc co {} {}".format(project, package)) + logger.info("osc co %s %s", project, package) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: - logger.error("checkout package error, {}".format(ret)) + logger.error("checkout package error, %s", ret) return False return True @@ -134,12 +134,12 @@ class OBSProxy(object): cmd = "cd {}; osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( package_path, repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) - logger.info("osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( - repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac)) + logger.info("osc build %s %s %s %s %s %s --no-verify --clean --noservice -M %s", + repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: - logger.error("build package error, {}".format(ret)) + logger.error("build package error, %s", ret) return False return True @@ -157,7 +157,7 @@ class OBSProxy(object): cmd = "osc api /build/{}/{}/{}/{}/_history".format(project, repo, arch, package) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.debug("list build history of package error, {}".format(ret)) + logger.debug("list build history of package error, %s", ret) return "" return "\n".join(out) diff --git a/src/proxy/requests_proxy.py b/src/proxy/requests_proxy.py index b88129f..87de829 100644 --- a/src/proxy/requests_proxy.py +++ b/src/proxy/requests_proxy.py @@ -38,9 +38,9 @@ def do_requests(method, url, querystring=None, body=None, auth=None, timeout=30, :return: """ try: - logger.debug("http requests, {} {} {}".format(method, url, timeout)) - logger.debug("querystring: {}".format(querystring)) - logger.debug("body: {}".format(body)) + logger.debug("http requests, %s %s %s", method, url, timeout) + logger.debug("querystring: %s", querystring) + logger.debug("body: %s", body) if method.lower() not in ["get", "post", "put", "delete"]: return -1 @@ -60,7 +60,7 @@ def do_requests(method, url, querystring=None, body=None, auth=None, timeout=30, else: rs = func(url, timeout=timeout) - logger.debug("status_code {}".format(rs.status_code)) + logger.debug("status_code %s", rs.status_code) if rs.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.no_content]: return 1 @@ -78,11 +78,11 @@ def do_requests(method, url, querystring=None, body=None, auth=None, timeout=30, return 0 except requests.exceptions.SSLError as e: - logger.warning("requests {} ssl exception, {}".format(url, e)) + logger.warning("requests %s ssl exception, %s", url, e) return -2 except requests.exceptions.Timeout as e: logger.warning("requests timeout") return 2 except requests.exceptions.RequestException as e: - logger.warning("requests exception, {}".format(e)) + logger.warning("requests exception, %s", e) return 3 diff --git a/src/tools/obs_package_build_history.py b/src/tools/obs_package_build_history.py index cd08637..ba9a8d0 100644 --- a/src/tools/obs_package_build_history.py +++ b/src/tools/obs_package_build_history.py @@ -46,7 +46,7 @@ class JobBuildHistory(object): try: root = ET.fromstring(history) except ParseError: - logger.exception("package: {}, build history: {}".format(package, history)) + logger.exception("package: %s, build history: %s", package, history) return {"package": package, "max": 0, "min": 0, "average": 0, "times": -1} duration = [int(ele.get("duration")) for ele in root.findall("entry")] @@ -74,11 +74,11 @@ class JobBuildHistory(object): for index in range(batch): works = [gevent.spawn(JobBuildHistory.get_package_job_duration, project, package, repo, arch) for package in packages[index * concurrency: (index + 1) * concurrency]] - logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) + logger.info("%s works, %s/%s ", len(works), index + 1, batch) gevent.joinall(works) for work in works: - logger.debug("{}: {}".format(work.value["package"], work.value)) - logger.info("{} ...done".format(work.value["package"])) + logger.debug("%s: %s", work.value["package"], work.value) + logger.info("{%s} ...done", work.value["package"]) rs.append(work.value) time.sleep(1) diff --git a/src/tools/obs_package_build_report.py b/src/tools/obs_package_build_report.py index cdb5084..d5cf53f 100644 --- a/src/tools/obs_package_build_report.py +++ b/src/tools/obs_package_build_report.py @@ -97,17 +97,17 @@ class ObsPackageBuildReport(object): # try: # branch = self.__class__.PROJECT_BRANCH_MAPPING[self._project] # except KeyError: -# logger.exception("project {} not support".format(self._project)) +# logger.exception("project %s not support", self._project) # return branch = "master" # get packages in project of state packages = OBSProxy.list_packages_of_state(self._project, self._state) - logger.info("project {} state {}, find {} packages".format(self._project, self._state, len(packages))) + logger.info("project %s state %s, find %s packages", self._project, self._state, len(packages)) # get last pr committer for index, package in enumerate(packages): - logger.info("{}: {}".format(index, package)) + logger.info("%s: %s", index, package) gp = GiteeProxy(self.GITEE_OWNER, package, gitee_api_token) committer = gp.get_last_pr_committer(branch) real_name = self._real_name_mapping.get(committer, "N/A") diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index c68c65c..e3be542 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -8,12 +8,18 @@ no_fmt_logger = logging.getLogger("no_fmt") def shell_cmd(cmd, inmsg=None): - logger.debug("exec cmd -- [{}]".format(cmd)) + """ + 创建子进程执行命令，返回执行结果 + :param cmd: 命令 + :param inmsg: 输入 + :return: + """ + logger.debug("exec cmd -- [%s]", cmd) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if inmsg: p.stdin.write(inmsg) out, err = p.communicate() - logger.debug("iret: {}, rs: {}, err: {}".format(p.returncode, out, err)) + logger.debug("iret: %s, rs: %s, err: %s", p.returncode, out, err) return p.returncode, out, err @@ -29,7 +35,7 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False :return: """ if cmd_verbose: - logger.debug("exec cmd -- {}".format(cmd)) + logger.debug("exec cmd -- %s", cmd) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if cap_in: @@ -51,13 +57,13 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False break if cap_out: - logger.debug("total {} lines output".format(len(out))) + logger.debug("total %s lines output", len(out)) ret = p.poll() err = None if ret: - logger.debug("return code {}".format(ret)) + logger.debug("return code %s", ret) while True: line= p.stderr.readline() if not line: diff --git a/test/ac/acl/package_yaml/test_check_repo.py b/test/ac/acl/package_yaml/test_check_repo.py index 8bf40f2..f1d7522 100644 --- a/test/ac/acl/package_yaml/test_check_repo.py +++ b/test/ac/acl/package_yaml/test_check_repo.py @@ -51,9 +51,9 @@ class TestGetReleaseTags(unittest.TestCase): with open(filepath, 'r') as yaml_data: # load yaml data result = yaml.safe_load(yaml_data) except IOError as e: - logging.warning("package yaml not exist. {}".format(str(e))) + logging.warning("package yaml not exist. %s", str(e)) except yaml.YAMLError as exc: - logging.warning("Error parsering YAML: {}".format(str(exc))) + logging.warning("Error parsering YAML: %s", str(exc)) finally: return result -- Gitee From 42673d358f725d69c94963b5c587266fc05b3cfc Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Mon, 8 Nov 2021 22:07:57 +0800 Subject: [PATCH 36/39] change deny_list to avoid self-compare --- src/conf/deny_list.sbom | 222 ++++++++++++++++++++++++++++------------ 1 file changed, 157 insertions(+), 65 deletions(-) diff --git a/src/conf/deny_list.sbom b/src/conf/deny_list.sbom index fd3a23e..7f1e3ff 100644 --- a/src/conf/deny_list.sbom +++ b/src/conf/deny_list.sbom @@ -61,165 +61,257 @@ "publisher": "src-openeuler", "name": "isula-transform" }, + { + "publisher": "openeuler", + "name": "kernel" + }, { "publisher": "openeuler-mirror", "name": "kernel" }, + { + "publisher": "src-openeuler", + "name": "kernel" + }, + { + "publisher": "openeuler", + "name": "async-libfuse" + }, { "publisher": "openeuler-mirror", "name": "async-libfuse" }, + { + "publisher": "src-openeuler", + "name": "async-libfuse" + }, + { + "publisher": "openeuler", + "name": "authz" + }, { "publisher": "openeuler-mirror", "name": "authz" }, + { + "publisher": "src-openeuler", + "name": "authz" + }, + { + "publisher": "openeuler", + "name": "clibcni" + }, { "publisher": "openeuler-mirror", "name": "clibcni" }, - { + { + "publisher": "src-openeuler", + "name": "clibcni" + }, + { + "publisher": "openeuler", + "name": "isula-build" + }, + { "publisher": "openeuler-mirror", "name": "isula-build" }, { + "publisher": "src-openeuler", + "name": "isula-build" + }, + { + "publisher": "openeuler", + "name": "iSulad" + }, + { "publisher": "openeuler-mirror", "name": "iSulad" }, { + "publisher": "src-openeuler", + "name": "iSulad" + }, + { + "publisher": "openeuler", + "name": "iSulad-img" + }, + { "publisher": "openeuler-mirror", "name": "iSulad-img" }, { + "publisher": "src-openeuler", + "name": "iSulad-img" + }, + { + "publisher": "openeuler", + "name": "iSula-libutils" + }, + { "publisher": "openeuler-mirror", "name": "iSula-libutils" }, { + "publisher": "src-openeuler", + "name": "iSula-libutils" + }, + { + "publisher": "openeuler", + "name": "isula-transform" + }, + { "publisher": "openeuler-mirror", "name": "isula-transform" }, { + "publisher": "src-openeuler", + "name": "isula-transform" + }, + { + "publisher": "openeuler", + "name": "lcr" + }, + { "publisher": "openeuler-mirror", "name": "lcr" }, { - "publisher": "openeuler-mirror", + "publisher": "src-openeuler", + "name": "lcr" + }, + { + "publisher": "openeuler", "name": "lib-shim-v2" }, - { + { "publisher": "openeuler-mirror", - "name": "lxcfs-tools" + "name": "lib-shim-v2" }, { - "publisher": "openeuler-mirror", - "name": "syscontainer-tools" + "publisher": "src-openeuler", + "name": "lib-shim-v2" }, { - "publisher": "openeuler-mirror", - "name": "A-Tune" + "publisher": "openeuler", + "name": "lxcfs-tools" }, - { + { "publisher": "openeuler-mirror", - "name": "A-Tune-Collector" + "name": "lxcfs-tools" }, - { - "publisher": "openeuler-mirror", - "name": "A-Tune-UI" + { + "publisher": "src-openeuler", + "name": "lxcfs-tools" }, - { - "publisher": "openeuler-mirror", - "name": "prefetch_tuning" + { + "publisher": "openeuler", + "name": "syscontainer-tools" }, - { + { "publisher": "openeuler-mirror", - "name": "wisdom-advisor" + "name": "syscontainer-tools" }, { - "publisher": "openeuler-mirror", - "name": "qemu" + "publisher": "src-openeuler", + "name": "syscontainer-tools" }, - { + { + "publisher": "openeuler", + "name": "A-Tune" + }, + { "publisher": "openeuler-mirror", - "name": "vmtop" + "name": "A-Tune" }, { "publisher": "src-openeuler", - "name": "kernel" + "name": "A-Tune" }, - { - "publisher": "src-openeuler", - "name": "async-libfuse" + { + "publisher": "openeuler", + "name": "A-Tune-Collector" }, - { - "publisher": "src-openeuler", - "name": "authz" + { + "publisher": "openeuler-mirror", + "name": "A-Tune-Collector" }, { "publisher": "src-openeuler", - "name": "clibcni" + "name": "A-Tune-Collector" }, - { - "publisher": "src-openeuler", - "name": "isula-build" + { + "publisher": "openeuler", + "name": "A-Tune-UI" }, - { - "publisher": "src-openeuler", - "name": "iSulad" + { + "publisher": "openeuler-mirror", + "name": "A-Tune-UI" }, { "publisher": "src-openeuler", - "name": "iSulad-img" + "name": "A-Tune-UI" }, - { - "publisher": "src-openeuler", - "name": "iSula-libutils" + { + "publisher": "openeuler", + "name": "prefetch_tuning" }, - { - "publisher": "src-openeuler", - "name": "isula-transform" + { + "publisher": "openeuler-mirror", + "name": "prefetch_tuning" }, { "publisher": "src-openeuler", - "name": "lcr" + "name": "prefetch_tuning" }, - { - "publisher": "src-openeuler", - "name": "lib-shim-v2" + { + "publisher": "openeuler", + "name": "wisdom-advisor" }, - { - "publisher": "src-openeuler", - "name": "lxcfs-tools" + { + "publisher": "openeuler-mirror", + "name": "wisdom-advisor" }, { "publisher": "src-openeuler", - "name": "syscontainer-tools" + "name": "wisdom-advisor" }, - { - "publisher": "src-openeuler", - "name": "A-Tune" + { + "publisher": "openeuler", + "name": "qemu" }, - { - "publisher": "src-openeuler", - "name": "A-Tune-Collector" + { + "publisher": "openeuler-mirror", + "name": "qemu" }, { "publisher": "src-openeuler", - "name": "A-Tune-UI" + "name": "qemu" + }, + { + "publisher": "openeuler", + "name": "vmtop" + }, + { + "publisher": "openeuler-mirror", + "name": "vmtop" }, { "publisher": "src-openeuler", - "name": "prefetch_tuning" + "name": "vmtop" }, { - "publisher": "src-openeuler", - "name": "wisdom-advisor" + "publisher": "openeuler", + "name": "kunpengsecl" }, { - "publisher": "src-openeuler", - "name": "qemu" + "ipublisher": "openeuler-mirror", + "name": "kunpengsecl" }, { "publisher": "src-openeuler", - "name": "vmtop" - } + "name": "kunpengsecl" + } ] } -- Gitee From 1cefb486f67d89a7c427b924c0e408d27590b4dd Mon Sep 17 00:00:00 2001 From: huangtianhua Date: Tue, 23 Nov 2021 08:32:43 +0000 Subject: [PATCH 37/39] Enable oepkg sp3 jenkins --- src/build/osc_build_k8s.py | 3 +++ src/build/related_rpm_package.py | 5 ++++- src/tools/obs_package_build_report.py | 5 ++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index be38c69..1804201 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -47,6 +47,9 @@ class SinglePackageBuild(object): "oepkg_openstack-common_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:common"], "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"], "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"], + "oepkg_openstack-common_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:common"], + "oepkg_openstack-queens_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:queens"], + "oepkg_openstack-rocky_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:rocky"] "openEuler-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3", "openEuler:20.03:LTS:SP3:Epol"] } diff --git a/src/build/related_rpm_package.py b/src/build/related_rpm_package.py index 97b891c..e07e5a5 100755 --- a/src/build/related_rpm_package.py +++ b/src/build/related_rpm_package.py @@ -44,7 +44,10 @@ class RelatedRpms(object): "oepkg_openstack-queens_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:queens"], "oepkg_openstack-common_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:common"], "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"], - "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"] + "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"], + "oepkg_openstack-common_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:common"], + "oepkg_openstack-rocky_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:rocky"], + "oepkg_openstack-queens_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:queens"] } def __init__(self, obs_addr, obs_repo_url, branch_name, package_arch): diff --git a/src/tools/obs_package_build_report.py b/src/tools/obs_package_build_report.py index d5cf53f..78e25e4 100644 --- a/src/tools/obs_package_build_report.py +++ b/src/tools/obs_package_build_report.py @@ -53,7 +53,10 @@ class ObsPackageBuildReport(object): "openEuler:20.03:LTS:SP2:oepkg:openstack:rocky": "oepkg_openstack-rocky_oe-20.03-LTS-SP2", "openEuler:20.03:LTS:Next:oepkg:openstack:common": "oepkg_openstack-common_oe-20.03-LTS-Next", "openEuler:20.03:LTS:Next:oepkg:openstack:queens": "oepkg_openstack-queens_oe-20.03-LTS-Next", - "openEuler:20.03:LTS:Next:oepkg:openstack:rocky": "oepkg_openstack-rocky_oe-20.03-LTS-Next" + "openEuler:20.03:LTS:Next:oepkg:openstack:rocky": "oepkg_openstack-rocky_oe-20.03-LTS-Next", + "openEuler:20.03:LTS:SP3:oepkg:openstack:common": "oepkg_openstack-common_oe-20.03-LTS-SP3", + "openEuler:20.03:LTS:SP3:oepkg:openstack:queens": "oepkg_openstack-queens_oe-20.03-LTS-SP3", + "openEuler:20.03:LTS:SP3:oepkg:openstack:rocky": "oepkg_openstack-rocky_oe-20.03-LTS-SP3" } GITEE_OWNER = "src-openeuler" -- Gitee From b96e32ffdd30c2edb83906bb04a31297138e017a Mon Sep 17 00:00:00 2001 From: huangtianhua Date: Thu, 25 Nov 2021 08:03:26 +0000 Subject: [PATCH 38/39] fix error config --- src/build/osc_build_k8s.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 1804201..e041d36 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -49,7 +49,7 @@ class SinglePackageBuild(object): "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"], "oepkg_openstack-common_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:common"], "oepkg_openstack-queens_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:queens"], - "oepkg_openstack-rocky_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:rocky"] + "oepkg_openstack-rocky_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:rocky"], "openEuler-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3", "openEuler:20.03:LTS:SP3:Epol"] } -- Gitee From 8d36b3a2931d94c2f59667606818dc7050486743 Mon Sep 17 00:00:00 2001 From: hht8 Date: Fri, 26 Nov 2021 09:56:50 +0800 Subject: [PATCH 39/39] support 22.03-LTS-Next --- src/build/osc_build_k8s.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index e041d36..c9b4d9b 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -50,7 +50,8 @@ class SinglePackageBuild(object): "oepkg_openstack-common_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:common"], "oepkg_openstack-queens_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:queens"], "oepkg_openstack-rocky_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:rocky"], - "openEuler-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3", "openEuler:20.03:LTS:SP3:Epol"] + "openEuler-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3", "openEuler:20.03:LTS:SP3:Epol"], + "openEuler-22.03-LTS-Next": ["openEuler:22.03:LTS:Next", "openEuler:22.03:LTS:Next:Epol"] } BUILD_IGNORED_GITEE_BRANCH = ["riscv"] -- Gitee