From 4f36c1bd10c3930a8e93a799a3d4236d482692df Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Mon, 15 Nov 2021 17:17:58 +0800 Subject: [PATCH 1/5] support select which spec files to build --- src/build/osc_build_k8s.py | 13 ++++++++----- src/proxy/obs_proxy.py | 11 ++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index e28ba34..dd9b32e 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -71,11 +71,12 @@ class SinglePackageBuild(object): """ return OBSProxy.list_repos_of_arch(project, self._package, self._arch, show_exclude=True) - def build_obs_repos(self, project, repos, work_dir, code_dir): + def build_obs_repos(self, project, repos, spec, work_dir, code_dir): """ build :param project: 项目名 :param repos: obs repo + :param spec: 指定spec文件 :param code_dir: 码云代码在本地路径 :param work_dir: :return: @@ -105,7 +106,7 @@ class SinglePackageBuild(object): continue root_build = repo["mpac"] in self.PACKAGES_USE_ROOT if not OBSProxy.build_package( - project, self._package, repo["repo"], self._arch, repo["mpac"], + project, self._package, repo["repo"], self._arch, spec, repo["mpac"], root_build=root_build, disable_cpio=True): logger.error("build {} ... failed".format(repo["repo"])) return 3 @@ -193,9 +194,10 @@ class SinglePackageBuild(object): return True - def build(self, work_dir, code_dir): + def build(self, spec, work_dir, code_dir): """ 入口 + :param spec: 指定spec文件 :param work_dir: obs工作目录 :param code_dir: 代码目录 :return: @@ -219,7 +221,7 @@ class SinglePackageBuild(object): logger.debug("build obs repos: {}".format(obs_repos)) has_any_repo_build = True - ret = self.build_obs_repos(project, obs_repos, work_dir, code_dir) + ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) if ret > 0: logger.debug("build run return {}".format(ret)) logger.error("build {} {} {} ... {}".format(project, self._package, self._arch, "failed")) @@ -254,6 +256,7 @@ def init_args(): parser.add_argument("-t", type=str, dest="account", help="gitee account") parser.add_argument("-o", type=str, dest="owner", default="src-openeuler", help="gitee owner") + parser.add_argument("--spec", type=str, dest="spec", default="", help="spec files") return parser.parse_args() @@ -316,7 +319,7 @@ if "__main__" == __name__: dd.set_attr_stime("spb.build.stime") spb = SinglePackageBuild(args.package, args.arch, args.branch) - rs = spb.build(args.workspace, args.code) + rs = spb.build(args.spec, args.workspace, args.code) dd.set_attr("spb.build.result", "failed" if rs else "successful") dd.set_attr_etime("spb.build.etime") diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index 6724748..9ccabc5 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -115,13 +115,14 @@ class OBSProxy(object): return True @staticmethod - def build_package(project, package, repo, arch, mpac, debug=False, root_build=False, disable_cpio=False): + def build_package(project, package, repo, arch, spec, mpac, debug=False, root_build=False, disable_cpio=False): """ build :param project: :param package: :param repo: :param arch: + :param spec: :param mpac: multibuild package :param debug: :return: @@ -130,11 +131,11 @@ class OBSProxy(object): root_opt = "--userootforbuild" if root_build else "" debuginfo_opt = "--disable-debuginfo" if not debug else "" disable_cpio_bulk = "--disable-cpio-bulk-download" if disable_cpio else "" - cmd = "cd {}; osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( - package_path, repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) + cmd = "cd {}; osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + package_path, repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) - logger.info("osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( - repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac)) + logger.info("osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac)) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: -- Gitee From a998f6880b1c8952a427ec23eb6c599e690132e6 Mon Sep 17 00:00:00 2001 From: hht8 Date: Thu, 14 Oct 2021 20:17:56 +0800 Subject: [PATCH 2/5] scanoss log optimization --- src/ac/acl/sca/check_sca.py | 66 +++++++++---------------------------- src/ac/framework/ac.py | 7 +--- src/requirements | 2 +- 3 files changed, 18 insertions(+), 57 deletions(-) diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index da04991..5d5be07 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -17,6 +17,7 @@ import os import shutil import logging +import json from src.proxy.git_proxy import GitProxy from src.ac.framework.ac_base import BaseCheck @@ -39,50 +40,22 @@ class CheckSCA(BaseCheck): """ super(CheckSCA, self).__init__(workspace, repo, conf) - self._work_diff_dir = os.path.join(workspace, "diff") # 目标目录，保存变更了的代码 - - def copy_diff_files_to_dest(self, files): - """ - 拷贝所有diff文件到目标目录 - :param files: 文件列表 - :return: - """ - for filepath in files: - try: - shutil.copy(os.path.join(self._work_dir, filepath), self._work_diff_dir) - except IOError: - logger.exception("copy {} to {} exception".format(filepath, self._work_diff_dir)) - - def save_scanoss_result(self, html): - """ - 保存结果到本地 - :param html: scanoss 结果，html格式 - :return: - """ - with open(self._scanoss_result_output, "w") as f: - f.write(html) - def check_scanoss(self): """ - scanoss工具检查代码片段引用 - https://osskb.org - https://github.com/scanoss/scanner.py - :return: + Obtain scanoss logs and result """ - gp = GitProxy(self._work_dir) - diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") - logger.debug("diff files: {}".format(diff_files)) - - self.copy_diff_files_to_dest(diff_files) - - blacklist_sbom = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../../../conf/deny_list.sbom")) - scan = ScanOSS(self._scanoss_api_key, self._scanoss_api_url, blacklist_sbom) - result = scan.scan(self._work_diff_dir) - + # Describes the reportUrl result jenkinsJobName jenkinsBuildNum prNo repoUrl of scanoss + try: + with open(self._scanoss_result_output, 'r') as f: + result_dirt = json.load(f) + except IOError: + logger.error("{} not found, make sure this file exists".format(self._scanoss_result_output)) + return FAILED + + result = result_dirt.get('result') + # 保存详细结果到web server - if not result: - self.save_scanoss_result(scan.html) - logger.warning("click {} view scanoss detail".format(self._scanoss_result_repo_path)) + logger.warning("click {} view scanoss detail".format(result_dirt.get('reportUrl'))) return SUCCESS if result else FAILED @@ -97,13 +70,6 @@ class CheckSCA(BaseCheck): logger.debug("args: {}, kwargs: {}".format(args, kwargs)) scanoss_conf = kwargs.get("scanoss", {}) - self._scanoss_api_key = scanoss_conf.get("api_key", "") - self._scanoss_api_url = scanoss_conf.get("api_url", "https://osskb.org/api/scan/direct") - self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") # 保存结果到本地文件 - self._scanoss_result_repo_path = scanoss_conf.get("repo_path", "-lost linker-") # 保存结果到web server的路径 - - _ = not os.path.exists(self._work_diff_dir) and os.mkdir(self._work_diff_dir) - try: - return self.start_check() - finally: - shutil.rmtree(self._work_diff_dir) + self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") + + return self.start_check() diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index f5387d2..a205dc9 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -202,12 +202,8 @@ def init_args(): parser.add_argument("-l", type=str, dest="trigger_link", help="job trigger link") # scanoss - parser.add_argument("--scanoss-api-key", type=str, dest="scanoss_api_key", help="scanoss api key") - parser.add_argument("--scanoss-api-url", type=str, dest="scanoss_api_url", - default="https://osskb.org/api/scan/direct", help="scanoss api url") parser.add_argument("--scanoss-output", type=str, dest="scanoss_output", default="scanoss_result", help="scanoss result output") - parser.add_argument("--scanoss-repo-path", type=str, dest="scanoss_repo", help="scanoss result repo path") parser.add_argument("--codecheck-api-key", type=str, dest="codecheck_api_key", help="codecheck api key") parser.add_argument("--codecheck-api-url", type=str, dest="codecheck_api_url", @@ -290,8 +286,7 @@ if "__main__" == __name__: gp.create_tags_of_pr(args.pr, "ci_processing") # scanoss conf - scanoss = {"api_key": args.scanoss_api_key, "api_url": args.scanoss_api_url, - "output": args.scanoss_output, "repo_path": args.scanoss_repo} + scanoss = {"output": args.scanoss_output} codecheck = {"pr_url": "https://gitee.com/{}/{}/pulls/{}".format(args.community, args.repo, args.pr), "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url, "codecheck_api_key": args.codecheck_api_key diff --git a/src/requirements b/src/requirements index d3fd573..7d34070 100644 --- a/src/requirements +++ b/src/requirements @@ -11,4 +11,4 @@ chardet kafka-python elasticsearch retrying -scanoss-scanner +scanoss -- Gitee From c5d645ee69c6a947c0c8582ff04e33a3e3bc8662 Mon Sep 17 00:00:00 2001 From: hcy1012 <690765764@qq.com> Date: Wed, 27 Oct 2021 08:30:55 +0000 Subject: [PATCH 3/5] =?UTF-8?q?update=20README.md.=20=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E8=8B=B1=E6=96=87=E8=AF=AD=E6=B3=95=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b9ed15e..70be875 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Introduction -This repository is used to store the jenkins scripts in openEuler Community. +This repository is used to store the Jenkins scripts in the openEuler community. ## License -- Gitee From e11156d20c7318fccd888fdf0de54e4c491fd8cf Mon Sep 17 00:00:00 2001 From: hcy1012 <690765764@qq.com> Date: Wed, 27 Oct 2021 08:32:22 +0000 Subject: [PATCH 4/5] =?UTF-8?q?add=20README-zh-cn.md.=20=E6=96=B0=E5=A2=9E?= =?UTF-8?q?=E4=B8=AD=E6=96=87=E7=89=88=E6=9C=ACREADME?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README-zh-cn.md | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 README-zh-cn.md diff --git a/README-zh-cn.md b/README-zh-cn.md new file mode 100644 index 0000000..c9525c3 --- /dev/null +++ b/README-zh-cn.md @@ -0,0 +1,9 @@ +# openeuler-jenkins + +## 简介 + +此仓库用来存放openEuler社区的Jenkins脚本。 + +## 许可证 + +详情请参考[LICENSE](https://gitee.com/openeuler/openeuler-jenkins/blob/ac397ce3e078937c700df6fb8de0e1b065ee4218/LICENSE)文件。 \ No newline at end of file -- Gitee From e83618829c1ca5fbfbc1d50ccb5e103177bb5714 Mon Sep 17 00:00:00 2001 From: zhengyaohui Date: Thu, 18 Nov 2021 22:42:53 +0800 Subject: [PATCH 5/5] solve logging-format-interolation(W1202) --- src/ac/acl/sca/check_sca.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index 5d5be07..362c1d9 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -49,13 +49,13 @@ class CheckSCA(BaseCheck): with open(self._scanoss_result_output, 'r') as f: result_dirt = json.load(f) except IOError: - logger.error("{} not found, make sure this file exists".format(self._scanoss_result_output)) + logger.error("%s not found, make sure this file exists", self._scanoss_result_output) return FAILED result = result_dirt.get('result') # 保存详细结果到web server - logger.warning("click {} view scanoss detail".format(result_dirt.get('reportUrl'))) + logger.warning("click %s view scanoss detail", result_dirt.get('reportUrl')) return SUCCESS if result else FAILED @@ -66,9 +66,9 @@ class CheckSCA(BaseCheck): :param kwargs: :return: """ - logger.info("check {} sca ...".format(self._repo)) + logger.info("check %s sca ...", self._repo) - logger.debug("args: {}, kwargs: {}".format(args, kwargs)) + logger.debug("args: %s, kwargs: %s", args, kwargs) scanoss_conf = kwargs.get("scanoss", {}) self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") -- Gitee