From 4d7244bba4726a6648def9934ac988107e4aee63 Mon Sep 17 00:00:00 2001 From: tan <1179903787@qq.com> Date: Wed, 13 Oct 2021 18:42:36 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E9=A1=B9=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ac/framework/ac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 9946b92..36d6e13 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -36,7 +36,7 @@ openeuler: entry: CheckSCA allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform", "kunpengsecl"] openlibing: - hint: check_openlibing + hint: code_check module: openlibing.check_code entry: CheckCode allow_list: ["pkgship", "kunpengsecl", "release-tools"] -- Gitee From 394769327e20a00d43f9dc602b592f148769a7a8 Mon Sep 17 00:00:00 2001 From: gaodi12 Date: Thu, 14 Oct 2021 15:59:59 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E6=9F=A5?= =?UTF-8?q?=E9=A1=B9=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/ac/framework/ac.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 36d6e13..7ff2457 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -36,7 +36,7 @@ openeuler: entry: CheckSCA allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform", "kunpengsecl"] openlibing: - hint: code_check + hint: code module: openlibing.check_code entry: CheckCode allow_list: ["pkgship", "kunpengsecl", "release-tools"] -- Gitee From dab8f63c85fba862a0e7cbb6b7bc06d65b345ef8 Mon Sep 17 00:00:00 2001 From: hht8 Date: Thu, 14 Oct 2021 20:17:56 +0800 Subject: [PATCH 3/3] scanoss log optimization --- src/ac/acl/sca/check_sca.py | 64 +++++++++++-------------------------- src/ac/framework/ac.py | 7 +--- src/requirements | 2 +- 3 files changed, 21 insertions(+), 52 deletions(-) diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index da04991..28d5fd1 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -17,6 +17,7 @@ import os import shutil import logging +import json from src.proxy.git_proxy import GitProxy from src.ac.framework.ac_base import BaseCheck @@ -39,50 +40,27 @@ class CheckSCA(BaseCheck): """ super(CheckSCA, self).__init__(workspace, repo, conf) - self._work_diff_dir = os.path.join(workspace, "diff") # 目标目录，保存变更了的代码 - - def copy_diff_files_to_dest(self, files): - """ - 拷贝所有diff文件到目标目录 - :param files: 文件列表 - :return: - """ - for filepath in files: - try: - shutil.copy(os.path.join(self._work_dir, filepath), self._work_diff_dir) - except IOError: - logger.exception("copy {} to {} exception".format(filepath, self._work_diff_dir)) - - def save_scanoss_result(self, html): - """ - 保存结果到本地 - :param html: scanoss 结果，html格式 - :return: - """ - with open(self._scanoss_result_output, "w") as f: - f.write(html) - def check_scanoss(self): """ - scanoss工具检查代码片段引用 - https://osskb.org - https://github.com/scanoss/scanner.py - :return: + Obtain scanoss logs and result """ - gp = GitProxy(self._work_dir) - diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") - logger.debug("diff files: {}".format(diff_files)) - - self.copy_diff_files_to_dest(diff_files) - - blacklist_sbom = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../../../conf/deny_list.sbom")) - scan = ScanOSS(self._scanoss_api_key, self._scanoss_api_url, blacklist_sbom) - result = scan.scan(self._work_diff_dir) - + try: + with open(self._scanoss_result_output, 'r') as f: + result_dirt = json.load(f) + except IOError: + logger.error("{} not found, make sure this file exists".format(self._scanoss_result_output)) + return FAILED + # Describes the reportUrl result jenkinsJobName jenkinsBuildNum prNo repoUrl of scanoss + try: + result = result_dirt.get('result') + except KeyError: + logger.error("result key value does not exist") # 保存详细结果到web server if not result: - self.save_scanoss_result(scan.html) - logger.warning("click {} view scanoss detail".format(self._scanoss_result_repo_path)) + try: + logger.warning("click {} view scanoss detail".format(result_dirt.get('reportUrl'))) + except KeyError: + logger.error("reportUrl key value does not exist") return SUCCESS if result else FAILED @@ -97,12 +75,8 @@ class CheckSCA(BaseCheck): logger.debug("args: {}, kwargs: {}".format(args, kwargs)) scanoss_conf = kwargs.get("scanoss", {}) - self._scanoss_api_key = scanoss_conf.get("api_key", "") - self._scanoss_api_url = scanoss_conf.get("api_url", "https://osskb.org/api/scan/direct") - self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") # 保存结果到本地文件 - self._scanoss_result_repo_path = scanoss_conf.get("repo_path", "-lost linker-") # 保存结果到web server的路径 - - _ = not os.path.exists(self._work_diff_dir) and os.mkdir(self._work_diff_dir) + self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") + try: return self.start_check() finally: diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index 6e32ebb..7789bca 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -202,12 +202,8 @@ def init_args(): parser.add_argument("-l", type=str, dest="trigger_link", help="job trigger link") # scanoss - parser.add_argument("--scanoss-api-key", type=str, dest="scanoss_api_key", help="scanoss api key") - parser.add_argument("--scanoss-api-url", type=str, dest="scanoss_api_url", - default="https://osskb.org/api/scan/direct", help="scanoss api url") parser.add_argument("--scanoss-output", type=str, dest="scanoss_output", default="scanoss_result", help="scanoss result output") - parser.add_argument("--scanoss-repo-path", type=str, dest="scanoss_repo", help="scanoss result repo path") parser.add_argument("--codecheck-api-url", type=str, dest="codecheck_api_url", default="http://124.71.75.234:8384/api/openlibing/codecheck/start", help="codecheck api url") @@ -289,8 +285,7 @@ if "__main__" == __name__: gp.create_tags_of_pr(args.pr, "ci_processing") # scanoss conf - scanoss = {"api_key": args.scanoss_api_key, "api_url": args.scanoss_api_url, - "output": args.scanoss_output, "repo_path": args.scanoss_repo} + scanoss = {"output": args.scanoss_output} codecheck = {"pr_url": "https://gitee.com/{}/{}/pulls/{}".format(args.community, args.repo, args.pr), "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url diff --git a/src/requirements b/src/requirements index d3fd573..7d34070 100644 --- a/src/requirements +++ b/src/requirements @@ -11,4 +11,4 @@ chardet kafka-python elasticsearch retrying -scanoss-scanner +scanoss -- Gitee