diff --git a/README-zh-cn.md b/README-zh-cn.md new file mode 100644 index 0000000000000000000000000000000000000000..c9525c30a95494513fdaa7659e88f7e4e2ddee45 --- /dev/null +++ b/README-zh-cn.md @@ -0,0 +1,9 @@ +# openeuler-jenkins + +## 简介 + +此仓库用来存放openEuler社区的Jenkins脚本。 + +## 许可证 + +详情请参考[LICENSE](https://gitee.com/openeuler/openeuler-jenkins/blob/ac397ce3e078937c700df6fb8de0e1b065ee4218/LICENSE)文件。 \ No newline at end of file diff --git a/README.md b/README.md index b9ed15e4f38e0b5bb26d1f51771bcc031627d08b..70be8755622182fddf5e0169c9c549fd84ca78b1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Introduction -This repository is used to store the jenkins scripts in openEuler Community. +This repository is used to store the Jenkins scripts in the openEuler community. ## License diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index da04991c38809f129cd8d3f30ff415023d72871c..362c1d9ecf549214d7fa12b724487bd11ff4d3bc 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -17,6 +17,7 @@ import os import shutil import logging +import json from src.proxy.git_proxy import GitProxy from src.ac.framework.ac_base import BaseCheck @@ -39,50 +40,22 @@ class CheckSCA(BaseCheck): """ super(CheckSCA, self).__init__(workspace, repo, conf) - self._work_diff_dir = os.path.join(workspace, "diff") # 目标目录,保存变更了的代码 - - def copy_diff_files_to_dest(self, files): - """ - 拷贝所有diff文件到目标目录 - :param files: 文件列表 - :return: - """ - for filepath in files: - try: - shutil.copy(os.path.join(self._work_dir, filepath), self._work_diff_dir) - except IOError: - logger.exception("copy {} to {} exception".format(filepath, self._work_diff_dir)) - - def save_scanoss_result(self, html): - """ - 保存结果到本地 - :param html: scanoss 结果,html格式 - :return: - """ - with open(self._scanoss_result_output, "w") as f: - f.write(html) - def check_scanoss(self): """ - scanoss工具检查代码片段引用 - https://osskb.org - https://github.com/scanoss/scanner.py - :return: + Obtain scanoss logs and result """ - gp = GitProxy(self._work_dir) - diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") - logger.debug("diff files: {}".format(diff_files)) - - self.copy_diff_files_to_dest(diff_files) - - blacklist_sbom = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../../../conf/deny_list.sbom")) - scan = ScanOSS(self._scanoss_api_key, self._scanoss_api_url, blacklist_sbom) - result = scan.scan(self._work_diff_dir) - + # Describes the reportUrl result jenkinsJobName jenkinsBuildNum prNo repoUrl of scanoss + try: + with open(self._scanoss_result_output, 'r') as f: + result_dirt = json.load(f) + except IOError: + logger.error("%s not found, make sure this file exists", self._scanoss_result_output) + return FAILED + + result = result_dirt.get('result') + # 保存详细结果到web server - if not result: - self.save_scanoss_result(scan.html) - logger.warning("click {} view scanoss detail".format(self._scanoss_result_repo_path)) + logger.warning("click %s view scanoss detail", result_dirt.get('reportUrl')) return SUCCESS if result else FAILED @@ -93,17 +66,10 @@ class CheckSCA(BaseCheck): :param kwargs: :return: """ - logger.info("check {} sca ...".format(self._repo)) + logger.info("check %s sca ...", self._repo) - logger.debug("args: {}, kwargs: {}".format(args, kwargs)) + logger.debug("args: %s, kwargs: %s", args, kwargs) scanoss_conf = kwargs.get("scanoss", {}) - self._scanoss_api_key = scanoss_conf.get("api_key", "") - self._scanoss_api_url = scanoss_conf.get("api_url", "https://osskb.org/api/scan/direct") - self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") # 保存结果到本地文件 - self._scanoss_result_repo_path = scanoss_conf.get("repo_path", "-lost linker-") # 保存结果到web server的路径 - - _ = not os.path.exists(self._work_diff_dir) and os.mkdir(self._work_diff_dir) - try: - return self.start_check() - finally: - shutil.rmtree(self._work_diff_dir) + self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") + + return self.start_check() diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index f5387d21a89df82da7278a0a9bad61011330a274..a205dc9eca283f6547f08ebca72a56daed4748e2 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -202,12 +202,8 @@ def init_args(): parser.add_argument("-l", type=str, dest="trigger_link", help="job trigger link") # scanoss - parser.add_argument("--scanoss-api-key", type=str, dest="scanoss_api_key", help="scanoss api key") - parser.add_argument("--scanoss-api-url", type=str, dest="scanoss_api_url", - default="https://osskb.org/api/scan/direct", help="scanoss api url") parser.add_argument("--scanoss-output", type=str, dest="scanoss_output", default="scanoss_result", help="scanoss result output") - parser.add_argument("--scanoss-repo-path", type=str, dest="scanoss_repo", help="scanoss result repo path") parser.add_argument("--codecheck-api-key", type=str, dest="codecheck_api_key", help="codecheck api key") parser.add_argument("--codecheck-api-url", type=str, dest="codecheck_api_url", @@ -290,8 +286,7 @@ if "__main__" == __name__: gp.create_tags_of_pr(args.pr, "ci_processing") # scanoss conf - scanoss = {"api_key": args.scanoss_api_key, "api_url": args.scanoss_api_url, - "output": args.scanoss_output, "repo_path": args.scanoss_repo} + scanoss = {"output": args.scanoss_output} codecheck = {"pr_url": "https://gitee.com/{}/{}/pulls/{}".format(args.community, args.repo, args.pr), "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url, "codecheck_api_key": args.codecheck_api_key diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index e28ba34184fe0b74c18fc7656f73458629545d04..dd9b32e9a733763b6c654bb4a38b62d6e95bc8c6 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -71,11 +71,12 @@ class SinglePackageBuild(object): """ return OBSProxy.list_repos_of_arch(project, self._package, self._arch, show_exclude=True) - def build_obs_repos(self, project, repos, work_dir, code_dir): + def build_obs_repos(self, project, repos, spec, work_dir, code_dir): """ build :param project: 项目名 :param repos: obs repo + :param spec: 指定spec文件 :param code_dir: 码云代码在本地路径 :param work_dir: :return: @@ -105,7 +106,7 @@ class SinglePackageBuild(object): continue root_build = repo["mpac"] in self.PACKAGES_USE_ROOT if not OBSProxy.build_package( - project, self._package, repo["repo"], self._arch, repo["mpac"], + project, self._package, repo["repo"], self._arch, spec, repo["mpac"], root_build=root_build, disable_cpio=True): logger.error("build {} ... failed".format(repo["repo"])) return 3 @@ -193,9 +194,10 @@ class SinglePackageBuild(object): return True - def build(self, work_dir, code_dir): + def build(self, spec, work_dir, code_dir): """ 入口 + :param spec: 指定spec文件 :param work_dir: obs工作目录 :param code_dir: 代码目录 :return: @@ -219,7 +221,7 @@ class SinglePackageBuild(object): logger.debug("build obs repos: {}".format(obs_repos)) has_any_repo_build = True - ret = self.build_obs_repos(project, obs_repos, work_dir, code_dir) + ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) if ret > 0: logger.debug("build run return {}".format(ret)) logger.error("build {} {} {} ... {}".format(project, self._package, self._arch, "failed")) @@ -254,6 +256,7 @@ def init_args(): parser.add_argument("-t", type=str, dest="account", help="gitee account") parser.add_argument("-o", type=str, dest="owner", default="src-openeuler", help="gitee owner") + parser.add_argument("--spec", type=str, dest="spec", default="", help="spec files") return parser.parse_args() @@ -316,7 +319,7 @@ if "__main__" == __name__: dd.set_attr_stime("spb.build.stime") spb = SinglePackageBuild(args.package, args.arch, args.branch) - rs = spb.build(args.workspace, args.code) + rs = spb.build(args.spec, args.workspace, args.code) dd.set_attr("spb.build.result", "failed" if rs else "successful") dd.set_attr_etime("spb.build.etime") diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index 6724748e32bab4e2540615dd2480702635482ed2..9ccabc56f678b4c754b46eaba8936b11b83d6bc2 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -115,13 +115,14 @@ class OBSProxy(object): return True @staticmethod - def build_package(project, package, repo, arch, mpac, debug=False, root_build=False, disable_cpio=False): + def build_package(project, package, repo, arch, spec, mpac, debug=False, root_build=False, disable_cpio=False): """ build :param project: :param package: :param repo: :param arch: + :param spec: :param mpac: multibuild package :param debug: :return: @@ -130,11 +131,11 @@ class OBSProxy(object): root_opt = "--userootforbuild" if root_build else "" debuginfo_opt = "--disable-debuginfo" if not debug else "" disable_cpio_bulk = "--disable-cpio-bulk-download" if disable_cpio else "" - cmd = "cd {}; osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( - package_path, repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) + cmd = "cd {}; osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + package_path, repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) - logger.info("osc build {} {} {} {} {} --no-verify --clean --noservice -M {}".format( - repo, arch, root_opt, debuginfo_opt, disable_cpio_bulk, mpac)) + logger.info("osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac)) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: diff --git a/src/requirements b/src/requirements index d3fd573f2d4135e8280d57b07b7cd8b4112a154b..7d34070afd36a0242b173d47d037f55727678ba0 100644 --- a/src/requirements +++ b/src/requirements @@ -11,4 +11,4 @@ chardet kafka-python elasticsearch retrying -scanoss-scanner +scanoss