diff --git a/README-zh-cn.md b/README-zh-cn.md new file mode 100644 index 0000000000000000000000000000000000000000..c9525c30a95494513fdaa7659e88f7e4e2ddee45 --- /dev/null +++ b/README-zh-cn.md @@ -0,0 +1,9 @@ +# openeuler-jenkins + +## 简介 + +此仓库用来存放openEuler社区的Jenkins脚本。 + +## 许可证 + +详情请参考[LICENSE](https://gitee.com/openeuler/openeuler-jenkins/blob/ac397ce3e078937c700df6fb8de0e1b065ee4218/LICENSE)文件。 \ No newline at end of file diff --git a/README.md b/README.md index b9ed15e4f38e0b5bb26d1f51771bcc031627d08b..70be8755622182fddf5e0169c9c549fd84ca78b1 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ ## Introduction -This repository is used to store the jenkins scripts in openEuler Community. +This repository is used to store the Jenkins scripts in the openEuler community. ## License diff --git a/changelogs/openEuler_jenkins_changelogs.md b/changelogs/openEuler_jenkins_changelogs.md new file mode 100644 index 0000000000000000000000000000000000000000..b1d61f7b8d07bfb5d23199f46ea79857611228eb --- /dev/null +++ b/changelogs/openEuler_jenkins_changelogs.md @@ -0,0 +1,23 @@ +# openEuler CI 门禁变更记录表 + +## 背景 +为规范openEuler CI 门禁配置管理，让每一次配置变更有迹可循，特制定改变更记录表，用于记录对门禁系统做出的每一次配置变更。包括但不限于Jenkins软甲升级、Jenkins插件安装、工程结构变更等。 + + +## 表格填写说明 + +**变更日期**：发起变更的具体日期，例如：“2021-08-01”； +**变更类型**：描述变更操作类型，如软件升级、插件安装等； +**变更原因**：描述该变更希望解决什么问题； +**变更说明**：描述变更的具体内容。如升级软件到什么版本，安装哪些插件； +**责任人**：该变更发起者； +**是否涉及存档**：该变更如果是软件修改或配置变更，通常会有文档或代码修改，修改前后的配置文档或代码对比需要存档，以便后续追溯； +**配置文件或代码路径**：如果“是否涉及存档”中填选“是”，则需要在当前仓库该Change Log Table的同级目录下以变更名称新建文件夹来保存相关的变更文档； + +## 变更记录 + +|序号|变更类型|变更原因|变更说明|责任人|是否涉及存档|配置文件或代码路径| +|--|--|--|--|--|--|--| +| 1 | 插件安装与升级 |支撑代码扫描工具落地 |因代码扫描工具scanoss需要在jenkins中安装http_request插件，所以安装插件HTTP Request Plugin 1.10版本，同时升级其依赖的插件Credentials Plugin 从2.4.1到2.5版本 |曹志、邓鹏、颜小兵 |否 | NULL | +| 2 | | | | | | +| 3 | | | | | | diff --git a/src/ac/README.md b/src/ac/README.md index cc370809d7fc7c74cb8c4d3fa831a817e0726107..7a18f158da18661494fdf780dfd47d9cb8c27098 100644 --- a/src/ac/README.md +++ b/src/ac/README.md @@ -1,28 +1,39 @@ # 门禁检查 ## 如何加入检查项 -1. 在ci_check/src/ac目录下新建文件夹 -2. 在ac_conf.yaml中增加配置项，可选 +1. 在ci_check/src/ac目录下新建文件夹放置检查项代码 +2. 在ac_conf.yaml中增加配置项 ### 配置文件说明 ```yaml 示例=> spec: # ac项目名称 - hint: check_spec # gitee中显示名，缺省使用check_+项目名称 + hint: check_spec # gitee中显示检查项名称，缺省使用check_+项目名称 module: spec.check_spec # ac项目模块名称，缺省使用"项目名称+check_+项目名称" - entry: Entry # ac项目入口，入口属性具备callable，缺省使用"run" + entry: Entry # ac项目入口类名称，继承BaseCheck类，可自定义__callable__方法 exclude: true # 忽略该项检查 ignored: [] # ac项目内忽略的检查项，就算失败也不影响最终ac项目结果 + allow_list: [] # 只有出现在allow_list的包才执行当前检查项 + deny_list:[] # 出现在deny_list的包不执行当前检查项 ``` ### entry实现模板 ```yaml -class Entry(object): +from src.ac.framework.ac_base import BaseCheck +from src.ac.framework.ac_result import FAILED, SUCCESS, WARNING + + +class Entry(BaseCheck): def __call__(self, *args, **kwargs): # do the work ... + + def check_case_a(self): + # do the check + + return SUCCESS ``` ### 检查结果 @@ -32,3 +43,12 @@ class Entry(object): | 0 | SUCCESS | :white_check_mark:| | 1 | WARNING | :bug: | | 2 | FAILED | :x:| + +## 支持的检查项 +| 检查项 | 目录 | 描述 | +| --- | --- | --- | +| spec文件 | spec | 检查homepage是否可以访问、版本号单调递增、检查补丁文件是否存在| +| 代码风格 | code | 检查压缩包文件、检查补丁是否可以使用、执行linter工具 | +| yaml文件 | package_yaml | | +| license检查 | package_license | | +| 代码片段检查 | sca | 目前只针对自研项目 | \ No newline at end of file diff --git a/src/ac/acl/code/check_code_style.py b/src/ac/acl/code/check_code_style.py index cb574b539577b90d94b12df390ddab4b2cc3a212..7dde4161beac67d4481da64158470af51e80850d 100644 --- a/src/ac/acl/code/check_code_style.py +++ b/src/ac/acl/code/check_code_style.py @@ -68,7 +68,7 @@ class CheckCodeStyle(BaseCheck): """ gp = GitProxy(self._work_dir) diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") - logger.debug("diff files: {}".format(diff_files)) + logger.debug("diff files: %s", diff_files) diff_code_files = [] # 仓库中变更的代码文件 diff_patch_code_files = [] # patch内的代码文件 @@ -77,7 +77,7 @@ class CheckCodeStyle(BaseCheck): diff_code_files.append(diff_file) elif GiteeRepo.is_patch_file(diff_file): patch_dir = self._gr.patch_dir_mapping.get(diff_file) - logger.debug("diff patch {} apply at dir {}".format(diff_file, patch_dir)) + logger.debug("diff patch %s apply at dir %s", diff_file, patch_dir) if patch_dir is not None: files_in_patch = gp.extract_files_path_of_patch(diff_file) patch_code_files = [os.path.join(patch_dir, file_in_patch) @@ -88,13 +88,13 @@ class CheckCodeStyle(BaseCheck): for code_file in patch_code_files if os.path.exists(code_file)]) - logger.debug("diff code files: {}".format(diff_code_files)) - logger.debug("diff patch code files: {}".format(diff_patch_code_files)) + logger.debug("diff code files: %s", diff_code_files) + logger.debug("diff patch code files: %s", diff_patch_code_files) rs_1 = self.check_file_under_work_dir(diff_code_files) - logger.debug("check_file_under_work_dir: {}".format(rs_1)) + logger.debug("check_file_under_work_dir: %s", rs_1) rs_2 = self.check_files_inner_patch(diff_patch_code_files) - logger.debug("check_files_inner_patch: {}".format(rs_2)) + logger.debug("check_files_inner_patch: %s", rs_2) return rs_1 + rs_2 @@ -131,10 +131,10 @@ class CheckCodeStyle(BaseCheck): elif GiteeRepo.is_c_cplusplus_file(file_path): rs = LinterCheck.check_c_cplusplus(file_path) else: - logger.error("error when arrive here, unsupport file {}".format(file_path)) + logger.error("error when arrive here, unsupport file %s", file_path) return SUCCESS - logger.info("Linter: {:<40} {}".format(file_path, rs)) + logger.info("Linter: %s %s", file_path, rs) if rs.get("F", 0) > 0: return FAILED @@ -150,7 +150,7 @@ class CheckCodeStyle(BaseCheck): :param kwargs: :return: """ - logger.info("check {} repo ...".format(self._repo)) + logger.info("check %s repo ...", self._repo) _ = not os.path.exists(self._work_tar_dir) and os.mkdir(self._work_tar_dir) try: diff --git a/src/ac/acl/openlibing/__init__.py b/src/ac/acl/openlibing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..60fc9c4fa75a9a286851448688e79fd222895e3e --- /dev/null +++ b/src/ac/acl/openlibing/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-08-03 +# Description: check code static +# ********************************************************************************** +""" \ No newline at end of file diff --git a/src/ac/acl/openlibing/check_code.py b/src/ac/acl/openlibing/check_code.py new file mode 100644 index 0000000000000000000000000000000000000000..d0585092b754901bfd0eaf39254d84beb69caec4 --- /dev/null +++ b/src/ac/acl/openlibing/check_code.py @@ -0,0 +1,130 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-08-03 +# Description: check code static +# ********************************************************************************** +""" + +import logging +import time + +from src.ac.framework.ac_base import BaseCheck +from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS +from src.proxy.requests_proxy import do_requests + +logger = logging.getLogger("ac") + + +class CheckCode(BaseCheck): + """ + code check + """ + + def __init__(self, workspace, repo, conf=None): + """ + + :param workspace: + :param repo: + :param conf: + """ + super(CheckCode, self).__init__(workspace, repo, conf) + + @staticmethod + def get_codecheck_result(pr_url, codecheck_api_url, codecheck_api_key): + """ + 通过api调用codecheck + """ + # get codecheck Api Token + codecheck_token_api_url = '{}/token/{}'.format(codecheck_api_url, codecheck_api_key) + token_resp = {} + rs = do_requests("get", codecheck_token_api_url, obj=token_resp) + if rs != 0 or token_resp.get("code", "") != "200": + logger.error("get dynamic token failed") + return 'false', {} + + token = token_resp.get("data") + data = {"pr_url": pr_url, "token": token} + response_content = {} + # 创建codecheck检查任务 + codecheck_task_api_url = "{}/task".format(codecheck_api_url) + rs = do_requests("get", codecheck_task_api_url, querystring=data, obj=response_content) + if rs != 0 or response_content.get('code', '') != '200': + logger.error("create codecheck task failed; %s", response_content.get('msg', '')) + return 'false', {} + + uuid = response_content.get('uuid') + task_id = response_content.get('task_id') + data = {"uuid": uuid, "token": token} + codecheck_status_api_url = '{}/{}/status'.format(codecheck_api_url, task_id) + current_time = 0 + logger.info("codecheck probably need to 3min") + # 定时3min + while current_time < 180: + time.sleep(10) + response_content = {} + # 检查codecheck任务的执行状态 + rs = do_requests("get", codecheck_status_api_url, querystring=data, obj=response_content) + if rs == 0 and response_content.get('code') == '100': + current_time = current_time + 10 + continue + else: + break + return rs, response_content + + def check_code(self): + """ + 开始进行codecheck检查 + """ + # 等待计算结果 + rs, response_content = self.get_codecheck_result(self._pr_url, self._codecheck_api_url, self._codecheck_api_key) + + # 判断是否计算完成 + if rs != 0: + return SUCCESS + + if response_content.get('msg') == 'success': + """ + # 返回结果 { + "code": "200", + "msg": "success", + "data": "http://{ip}:{port}/inc/{projectId}/reports/{taskId}/detail" 一个可以看到codecheck检查结果详情的地址 + "state": "pass(通过)/no pass(不通过)" + } + """ + logger.warning("click %s view code check detail", response_content.get('data')) + # 只有codecheck完成且codecheck检查的代码中存在bug，返回检查项失败的结果，以detail结尾，会显示具体的代码bug所在位置。 + if response_content.get("state") == "no pass": + return FAILED + else: + logger.error("code check failed, info : %s", response_content.get('msg')) + + return SUCCESS + + def __call__(self, *args, **kwargs): + """ + 入口函数 + :param args: + :param kwargs: + :return: + """ + logger.info("check %s code ...", self._repo) + logger.debug("args: %s, kwargs: %s", args, kwargs) + codecheck_conf = kwargs.get("codecheck", {}) + + self._pr_url = codecheck_conf.get("pr_url", "") + self._pr_number = codecheck_conf.get("pr_number", "") + self._codecheck_api_url = codecheck_conf.get("codecheck_api_url", "") + self._codecheck_api_key = codecheck_conf.get('codecheck_api_key', "") + + return self.start_check() diff --git a/src/ac/acl/package_license/check_license.py b/src/ac/acl/package_license/check_license.py index fad13eecc65e5589cc787f5408849b2404b02262..669c139235c6b872a9055824c6f661a8cd3e7d88 100644 --- a/src/ac/acl/package_license/check_license.py +++ b/src/ac/acl/package_license/check_license.py @@ -87,14 +87,15 @@ class CheckLicense(BaseCheck): check whether the license in spec file and in src file is same :return """ - if self._pkg_license.check_licenses_is_same(self._license_in_spec, self._license_in_src): - logger.info("licenses in src:{} and in spec:{} are same".format(self._license_in_src, - self._license_in_spec)) + if self._pkg_license.check_licenses_is_same(self._license_in_spec, self._license_in_src, + self._pkg_license._later_support_license): + logger.info("licenses in src:%s and in spec:%s are same", self._license_in_src, + self._license_in_spec) return SUCCESS else: - logger.error("licenses in src:{} and in spec:{} are not same".format(self._license_in_src, - self._license_in_spec)) - return FAILED + logger.error("licenses in src:%s and in spec:%s are not same", self._license_in_src, + self._license_in_spec) + return WARNING def __call__(self, *args, **kwargs): """ @@ -103,7 +104,7 @@ class CheckLicense(BaseCheck): :param kwargs: :return: """ - logger.info("check {} license ...".format(self._repo)) + logger.info("check %s license ...", self._repo) _ = not os.path.exists(self._work_tar_dir) and os.mkdir(self._work_tar_dir) self._gr.decompress_all() # decompress all compressed file into work_tar_dir diff --git a/src/ac/acl/package_license/config/Licenses.yaml b/src/ac/acl/package_license/config/Licenses.yaml index f0ba7897fe0fd6b8f4ebe376ccbee099fe654b0b..894e300dec8127554532f39221a8b2ec1f84d734 100644 --- a/src/ac/acl/package_license/config/Licenses.yaml +++ b/src/ac/acl/package_license/config/Licenses.yaml @@ -637,9 +637,8 @@ Software Licenses: - Artistic License 1.0 w/clause 8 identifier: Artistic-1.0-cl8 - alias: + - The "Artistic License" - Artistic License 1.0 (Perl) - - (perl) - - "license: perl" identifier: Artistic-1.0-Perl - alias: - Lawrence Berkeley National Labs BSD variant license @@ -779,6 +778,10 @@ Software Licenses: - AFLv2.1 identifier: AFL-2.1 Free Licenses: + - alias: + - (perl) + - "license: perl" + identifier: Perl License - alias: - BSD Zero Clause License - Zero-Clause BSD @@ -1235,6 +1238,7 @@ Software Licenses: - alias: - Creative Commons Zero v1.0 Universal - CC0 + - CC0 1.0 Universal identifier: CC0-1.0 - alias: - TCL/TK License diff --git a/src/ac/acl/package_license/config/later_support_license.yaml b/src/ac/acl/package_license/config/later_support_license.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff1b5e11fa7cabb5d37cf885db530707126d2439 --- /dev/null +++ b/src/ac/acl/package_license/config/later_support_license.yaml @@ -0,0 +1,19 @@ +GPL: + versions: + - "1.0" + - "2.0" + - "3.0" +LGPL: + versions: + - "2.0" + - "2.1" + - "3.0" +AGPL: + versions: + - "1.0" + - "3.0" +GFDL: + versions: + - "1.1" + - "1.2" + - "1.3" \ No newline at end of file diff --git a/src/ac/acl/package_license/package_license.py b/src/ac/acl/package_license/package_license.py index 1735b4b969c0106caaf0a191484235c19f71e052..24a17c4d89250769044aec49ee3086bdb2e9cc13 100644 --- a/src/ac/acl/package_license/package_license.py +++ b/src/ac/acl/package_license/package_license.py @@ -54,10 +54,14 @@ class PkgLicense(object): LICENSE_YAML_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), "config", "Licenses.yaml") + LATER_SUPPORT_LICENSE_PATH = os.path.join(os.path.dirname(os.path.realpath(__file__)), + "config", + "later_support_license.yaml") def __init__(self): self._white_black_list = {} self._license_translation = {} + self._later_support_license = {} def load_config(self): """ @@ -80,6 +84,12 @@ class PkgLicense(object): except yaml.YAMLError as e: logger.exception("yaml load error: %s", str(e)) return + with open(self.LATER_SUPPORT_LICENSE_PATH, "r") as f: + try: + self._later_support_license = yaml.safe_load(f) + except yaml.YAMLError as e: + logger.exception("yaml load error: %s", str(e)) + return self._parse_tag_license(data["Software Licenses"]["Not Free Licenses"], "black") self._parse_tag_license(data["Software Licenses"]["Free Licenses"], @@ -157,11 +167,11 @@ class PkgLicense(object): for filename in filenames: if (filename.lower() in self.LICENSE_FILE_TARGET or self.LICENSE_TARGET_PAT.search(filename.lower())): - logger.info("scan the license target file: %s", filename) + logger.info("scan the license target file: %s", os.path.join(root, filename).replace(srcdir, "")) licenses_in_file.update( self.scan_licenses( os.path.join(root, filename))) - logger.info("all licenses from src: %s", ", ".join([data.encode("utf-8") for data in licenses_in_file])) + logger.info("all licenses from src: %s", ", ".join([data for data in licenses_in_file])) return licenses_in_file def scan_licenses(self, copying): @@ -215,9 +225,22 @@ class PkgLicense(object): return PkgLicense._decode_str(data, chardet.detect(data)["encoding"]) @staticmethod - def check_licenses_is_same(licenses_for_spec, licenses_for_source_files): + def check_licenses_is_same(licenses_for_spec, licenses_for_source_files, later_support_license): """ Check if the licenses from SPEC is the same as the licenses from LICENSE file. if same, return True. if not same return False. """ - return licenses_for_spec.issuperset(licenses_for_source_files) \ No newline at end of file + all_licenses_for_spec = set() + for license in licenses_for_spec: + if "-or-later" in license: + [l, v] = license.split("-or-later")[0].split("-", 1) + if l not in later_support_license: + all_licenses_for_spec.add(license) + continue + for version in later_support_license[l]["versions"]: + if version >= v: + all_licenses_for_spec.add(f"{l}-{version}-or-later") + all_licenses_for_spec.add(f"{l}-{version}-only") + else: + all_licenses_for_spec.add(license) + return all_licenses_for_spec.issuperset(licenses_for_source_files) \ No newline at end of file diff --git a/src/ac/acl/package_yaml/check_repo.py b/src/ac/acl/package_yaml/check_repo.py index 9541ea3923d2086cb0aa8bf868545481a2ac9c5a..11cdd96010439cf84cf0d3c471692584b268d50e 100644 --- a/src/ac/acl/package_yaml/check_repo.py +++ b/src/ac/acl/package_yaml/check_repo.py @@ -22,7 +22,7 @@ import logging import re -import urlparse +import urllib.parse as urlparse import requests import json import subprocess @@ -72,7 +72,7 @@ class DefaultReleaseTags(AbsReleaseTags): 通过url获取上游社区的release tags return: list """ - logging.info("unsupported version control: {}".format(self.version_control)) + logging.info("unsupported version control: %s", self.version_control) return [] @@ -118,7 +118,7 @@ class HttpReleaseTagsMixin(object): response = requests.get(url, headers=headers, timeout=timeout) need_redirect, new_url, cookies = self.get_redirect_resp(url, response) if tldextract.extract(url).domain != tldextract.extract(new_url).domain: # 判断域名是否一致 预防csrf攻击 - logging.warning("domain of redirection link is different: {}".format(new_url)) + logging.warning("domain of redirection link is different: %s", new_url) return "" if need_redirect: cookie_dict = {} @@ -128,13 +128,13 @@ class HttpReleaseTagsMixin(object): url = new_url response = requests.get(url, headers=headers, cookies=cookie_dict, timeout=timeout) except requests.exceptions.SSLError as e: - logging.warning("requests {} ssl exception, {}".format(url, e)) + logging.warning("requests %s ssl exception, %s", url, e) return "" except requests.exceptions.Timeout as e: logging.warning("requests timeout") return "" except requests.exceptions.RequestException as e: - logging.warning("requests exception, {}".format(e)) + logging.warning("requests exception, %s", e) return "" return response @@ -156,7 +156,7 @@ class HgReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -170,7 +170,7 @@ class HgReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): temp_tags.sort(reverse=True, key=lambda x: x["date"][0]) release_tags = [tag["tag"] for tag in temp_tags] except Exception as e: - logging.error("exception, {}".format(e)) + logging.error("exception, %s", e) return [] return release_tags @@ -192,7 +192,7 @@ class HgRawReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -220,11 +220,13 @@ class MetacpanReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] response = self.get_request_response(url) + if not response: + return [] resp_lines = response.text.splitlines() release_tags = [] tag_condition = "value=\"/release" @@ -256,7 +258,7 @@ class PypiReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -265,7 +267,7 @@ class PypiReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): tags_json = response.json() release_tags = [tag for tag in tags_json.get("releases")] except Exception as e: - logging.error("exception, {}".format(e)) + logging.error("exception, %s", e) return [] return release_tags @@ -287,7 +289,7 @@ class RubygemReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -299,7 +301,7 @@ class RubygemReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): if element.get("number"): release_tags.append(element.get("number")) except Exception as e: - logging.error("exception, {}".format(e)) + logging.error("exception, %s", e) return [] return release_tags @@ -321,13 +323,15 @@ class GnuftpReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] response = self.get_request_response(url) pattern = re.compile("href=\"(.*)\">(.*)") release_tags = [] + if not response: + return [] for line in response.text.splitlines(): search_result = pattern.search(line) if search_result: @@ -352,7 +356,7 @@ class FtpReleaseTags(AbsReleaseTags, HttpReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -378,7 +382,7 @@ class CmdReleaseTagsMixin(object): sub_proc = subprocess.Popen(cmd_list, stdout=subprocess.PIPE) response = sub_proc.stdout.read().decode("utf-8") if sub_proc.wait(): - logging.warning("{cmd} > encount errors".format(cmd=" ".join(cmd_list))) + logging.warning("%s > encount errors", " ".join(cmd_list)) return response @@ -407,7 +411,7 @@ class SvnReleaseTags(AbsReleaseTags, CmdReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get svn tags".format(repo=url)) + logging.debug("%s : get svn tags", url) if not url: logging.warning("illegal url: \"\"") return [] @@ -461,7 +465,7 @@ class GitReleaseTags(AbsReleaseTags, CmdReleaseTagsMixin): return: list """ url = self.url(repo) - logging.debug("{repo} : get {vc} tags".format(repo=url, vc=self.version_control)) + logging.debug("%s : get %s tags", url, self.version_control) if not url: logging.warning("illegal url: \"\"") return [] @@ -537,4 +541,4 @@ class ReleaseTagsFactory(object): return: class """ release_tags = ReleaseTagsFactory.VERSION_CTRL_GETTER_MAPPING.get(version_control, DefaultReleaseTags) - return release_tags(version_control) \ No newline at end of file + return release_tags(version_control) diff --git a/src/ac/acl/package_yaml/check_yaml.py b/src/ac/acl/package_yaml/check_yaml.py index 62f5d0b01d244514b43d9090ec9ad93d40c4a3d9..8ac90cc8396fecdccf5dfe228f13b6d9e99cf0f5 100644 --- a/src/ac/acl/package_yaml/check_yaml.py +++ b/src/ac/acl/package_yaml/check_yaml.py @@ -73,7 +73,7 @@ class CheckPackageYaml(BaseCheck): for change_file in diff_files: if change_file == package_yaml: - logger.debug("diff files: {}".format(diff_files)) + logger.debug("diff files: %s", diff_files) return True return False @@ -93,16 +93,16 @@ class CheckPackageYaml(BaseCheck): with open(os.path.join(self._work_dir, yaml_path), 'r') as yaml_data: # load yaml data self._yaml_content = yaml.safe_load(yaml_data) except IOError as e: - logging.warning("package yaml not exist. {}".format(str(e))) + logging.warning("package yaml not exist. %s", str(e)) return WARNING except yaml.YAMLError as exc: - logging.warning("Error parsering YAML: {}".format(str(exc))) + logging.warning("Error parsering YAML: %s", str(exc)) return WARNING result = SUCCESS for keyword in self.PACKAGE_YAML_NEEDED_KEY: if keyword not in self._yaml_content: - logger.error("yaml field {} missing".format(keyword)) + logger.error("yaml field %s missing", keyword) self._is_standard = True result = WARNING return result @@ -129,7 +129,7 @@ class CheckPackageYaml(BaseCheck): tags = release_tags.get_tags(sr) if not tags: - logger.warning("failed to get version by yaml, version_control: {t1}, src_repo: {t2}".format(t1=vc, t2=sr)) + logger.warning("failed to get version by yaml, version_control: %s, src_repo: %s", vc, sr) return WARNING return SUCCESS @@ -154,9 +154,9 @@ class CheckPackageYaml(BaseCheck): if not src_url: src_url = self._spec.get_source("Source") vc = self.VERSION_CTRL_TRANS.get(vc, vc) # 对特殊的版本控制对应的域名进行转换 - logger.debug("version control: {vctrl} source url: {url}".format(vctrl=vc, url=src_url)) + logger.debug("version control: %s source url: %s", vc, src_url) if vc not in src_url: # 通过判断版本控制字段是否在主页url中 判断一致性 - logger.warning("{vc} is not in url: {url}".format(vc=vc, url=src_url)) + logger.warning("%s is not in url: %s", vc, src_url) return WARNING return SUCCESS @@ -196,14 +196,15 @@ class CheckPackageYaml(BaseCheck): src_url = self._spec.get_source("Source0") if not src_url: src_url = self._spec.get_source("Source") - logger.debug("software name: {name} source url: {url}".format(name=software_name, url=src_url)) + logger.debug("software name: %s source url: %s", software_name, src_url) if software_name not in src_url: - logger.warning("{name} is not in source0: {url}".format(name=software_name, url=src_url)) + logger.warning("%s is not in source0: %s", software_name, src_url) return WARNING return SUCCESS def __call__(self, *args, **kwargs): - logger.info("check {} yaml ...".format(self._repo)) + logger.info("check %s yaml ...", self._repo) self._yaml_changed = self.is_change_package_yaml() # yaml文件变更 进行检查 - return self.start_check_with_order("fields", "repo_domain", "repo_name", "repo") + # 因门禁系统限制外网访问权限，将涉及外网访问的检查功能check_repo暂时关闭 + return self.start_check_with_order("fields", "repo_domain", "repo_name") diff --git a/src/ac/acl/sca/check_sca.py b/src/ac/acl/sca/check_sca.py index d4d9af9e72f6beb6d8a3eb7ac42354bfc128e17b..362c1d9ecf549214d7fa12b724487bd11ff4d3bc 100644 --- a/src/ac/acl/sca/check_sca.py +++ b/src/ac/acl/sca/check_sca.py @@ -17,6 +17,7 @@ import os import shutil import logging +import json from src.proxy.git_proxy import GitProxy from src.ac.framework.ac_base import BaseCheck @@ -39,50 +40,22 @@ class CheckSCA(BaseCheck): """ super(CheckSCA, self).__init__(workspace, repo, conf) - self._work_diff_dir = os.path.join(workspace, "diff") # 目标目录，保存变更了的代码 - - def copy_diff_files_to_dest(self, files): - """ - 拷贝所有diff文件到目标目录 - :param files: 文件列表 - :return: - """ - for filepath in files: - try: - shutil.copy(os.path.join(self._work_dir, filepath), self._work_diff_dir) - except IOError: - logger.exception("copy {} to {} exception".format(filepath, self._work_diff_dir)) - - def save_scanoss_result(self, html): - """ - 保存结果到本地 - :param html: scanoss 结果，html格式 - :return: - """ - with open(self._scanoss_result_output, "w") as f: - f.write(html) - def check_scanoss(self): """ - scanoss工具检查代码片段引用 - https://osskb.org - https://github.com/scanoss/scanner.py - :return: + Obtain scanoss logs and result """ - gp = GitProxy(self._work_dir) - diff_files = gp.diff_files_between_commits("HEAD~1", "HEAD~0") - logger.debug("diff files: {}".format(diff_files)) - - self.copy_diff_files_to_dest(diff_files) - - blacklist_sbom = os.path.realpath(os.path.join(os.path.realpath(__file__), "../../../../conf/deny_list.sbom")) - scan = ScanOSS(self._scanoss_api_key, self._scanoss_api_url, blacklist_sbom) - result = scan.scan(self._work_diff_dir) - + # Describes the reportUrl result jenkinsJobName jenkinsBuildNum prNo repoUrl of scanoss + try: + with open(self._scanoss_result_output, 'r') as f: + result_dirt = json.load(f) + except IOError: + logger.error("%s not found, make sure this file exists", self._scanoss_result_output) + return FAILED + + result = result_dirt.get('result') + # 保存详细结果到web server - if not result: - self.save_scanoss_result(scan.html) - logger.warning("click to view scanoss detail: {}".format(self._scanoss_result_repo_path)) + logger.warning("click %s view scanoss detail", result_dirt.get('reportUrl')) return SUCCESS if result else FAILED @@ -93,17 +66,10 @@ class CheckSCA(BaseCheck): :param kwargs: :return: """ - logger.info("check {} sca ...".format(self._repo)) + logger.info("check %s sca ...", self._repo) - logger.debug("args: {}, kwargs: {}".format(args, kwargs)) + logger.debug("args: %s, kwargs: %s", args, kwargs) scanoss_conf = kwargs.get("scanoss", {}) - self._scanoss_api_key = scanoss_conf.get("api_key", "") - self._scanoss_api_url = scanoss_conf.get("api_url", "https://osskb.org/api/scan/direct") - self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") # 保存结果到本地文件 - self._scanoss_result_repo_path = scanoss_conf.get("repo_path", "-lost linker-") # 保存结果到web server的路径 - - _ = not os.path.exists(self._work_diff_dir) and os.mkdir(self._work_diff_dir) - try: - return self.start_check() - finally: - shutil.rmtree(self._work_diff_dir) + self._scanoss_result_output = scanoss_conf.get("output", "scanoss_result") + + return self.start_check() diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index 6502e91481a280afcc6158b4cad149a9b4d9d786..540d41fb01b819991d57c818378ea1413f97098d 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -53,7 +53,7 @@ class CheckSpec(BaseCheck): package_yaml = "{}.yaml".format(self._repo) # package yaml file name if len(diff_files) == 1 and diff_files[0] == package_yaml: - logger.debug("diff files: {}".format(diff_files)) + logger.debug("diff files: %s", diff_files) return True return False @@ -94,7 +94,7 @@ class CheckSpec(BaseCheck): # if lts branch, version update is forbidden if self._is_lts_branch(): - logger.debug("lts branch {}".format(self._tbranch)) + logger.debug("lts branch %s", self._tbranch) if RPMSpecAdapter.compare_version(self._spec.version, spec_o.version) == 1: logger.error("version update of lts branch is forbidden") return FAILED @@ -106,8 +106,8 @@ class CheckSpec(BaseCheck): logger.debug("revert commit") return SUCCESS - logger.error("current version: {}-r{}, last version: {}-r{}".format( - self._spec.version, self._spec.release, spec_o.version, spec_o.release)) + logger.error("current version: %s-r%s, last version: %s-r%s", + self._spec.version, self._spec.release, spec_o.version, spec_o.release) return FAILED def check_homepage(self, timeout=30, retrying=3, interval=1): @@ -119,11 +119,11 @@ class CheckSpec(BaseCheck): :return: """ homepage = self._spec.url - logger.debug("homepage: {}".format(homepage)) + logger.debug("homepage: %s", homepage) if not homepage: return SUCCESS - for _ in xrange(retrying): + for _ in range(retrying): if 0 == do_requests("get", homepage, timeout=timeout): return SUCCESS time.sleep(interval) @@ -137,15 +137,15 @@ class CheckSpec(BaseCheck): """ patches_spec = set(self._spec.patches) patches_file = set(self._gr.patch_files_not_recursive()) - logger.debug("spec patches: {}".format(patches_spec)) - logger.debug("file patches: {}".format(patches_file)) + logger.debug("spec patches: %s", patches_spec) + logger.debug("file patches: %s", patches_file) result = SUCCESS for patch in patches_spec - patches_file: - logger.error("patch {} lost".format(patch)) + logger.error("patch %s lost", patch) result = FAILED for patch in patches_file - patches_spec: - logger.warning("patch {} redundant".format(patch)) + logger.warning("patch %s redundant", patch) return result @@ -164,7 +164,7 @@ class CheckSpec(BaseCheck): content = "x86-64" if content is not None: - logger.info("exclusive arch \"{}\"".format(content)) + logger.info("exclusive arch \"%s\"", content) try: with open("exclusive_arch", "w") as f: f.write(content) @@ -189,7 +189,7 @@ class CheckSpec(BaseCheck): "last_version": spec.version, "last_release": spec.release, "compare_version": compare_version, "compare_release": compare_release, "compare": compare} - logger.info("{}".format(rs)) + logger.info("%s", rs) try: with open("pkgship_notify", "w") as f: yaml.safe_dump(rs, f) @@ -197,8 +197,9 @@ class CheckSpec(BaseCheck): logger.exception("save pkgship exception") def __call__(self, *args, **kwargs): - logger.info("check {} spec ...".format(self._repo)) + logger.info("check %s spec ...", self._repo) self._ex_exclusive_arch() self._tbranch = kwargs.get("tbranch", None) - return self.start_check() + # 因门禁系统限制外网访问权限，将涉及外网访问的检查功能check_homepage暂时关闭 + return self.start_check_with_order("version", "patches") diff --git a/src/ac/common/gitee_repo.py b/src/ac/common/gitee_repo.py index 3432d4f579c83e9305b5f8eead33b17c835d510e..c24ca09039c0c6bbe7d01f2d8a0d535c7770299d 100644 --- a/src/ac/common/gitee_repo.py +++ b/src/ac/common/gitee_repo.py @@ -53,16 +53,16 @@ class GiteeRepo(object): for filename in filenames: rel_file_path = os.path.join(dirpath, filename).replace(self._work_dir, "").lstrip("/") if self.is_compress_file(filename): - logger.debug("find compress file: {}".format(rel_file_path)) + logger.debug("find compress file: %s", rel_file_path) self._compress_files.append(rel_file_path) elif self.is_patch_file(filename): - logger.debug("find patch file: {}".format(rel_file_path)) + logger.debug("find patch file: %s", rel_file_path) self._patch_files.append(rel_file_path) elif self.is_spec_file(filename): - logger.debug("find spec file: {}".format(rel_file_path)) + logger.debug("find spec file: %s", rel_file_path) spec_files.append(filename) elif self.is_package_yaml_file(filename): - logger.debug("find yaml file: {}".format(rel_file_path)) + logger.debug("find yaml file: %s", rel_file_path) self.yaml_file = rel_file_path def guess_real_spec_file(): @@ -103,11 +103,13 @@ class GiteeRepo(object): :return: """ if self._is_compress_zip_file(file_path): - decompress_cmd = "cd {}; unzip -d {} {}".format(self._work_dir, self._decompress_dir, file_path) + decompress_cmd = "cd {}; timeout 120s unzip -o -d {} {}".format( + self._work_dir, self._decompress_dir, file_path) elif self._is_compress_tar_file(file_path): - decompress_cmd = "cd {}; tar -C {} -xavf {}".format(self._work_dir, self._decompress_dir, file_path) + decompress_cmd = "cd {}; timeout 120s tar -C {} -xavf {}".format( + self._work_dir, self._decompress_dir, file_path) else: - logger.warning("unsupport compress file: {}".format(file_path)) + logger.warning("unsupport compress file: %s", file_path) return False ret, _, _ = shell_cmd_live(decompress_cmd) @@ -134,20 +136,20 @@ class GiteeRepo(object): :param patch: 补丁 :param max_leading: leading path """ - logger.debug("apply patch {}".format(patch)) + logger.debug("apply patch %s", patch) for patch_dir in [filename for filename in os.listdir(self._decompress_dir) if os.path.isdir(os.path.join(self._decompress_dir, filename))] + ["."]: if patch_dir.startswith(".git"): continue - for leading in xrange(max_leading + 1): - logger.debug("try dir {} -p{}".format(patch_dir, leading)) + for leading in range(max_leading + 1): + logger.debug("try dir %s -p%s", patch_dir, leading) if GitProxy.apply_patch_at_dir(os.path.join(self._decompress_dir, patch_dir), os.path.join(self._work_dir, patch), leading): logger.debug("patch success") self.patch_dir_mapping[patch] = os.path.join(self._decompress_dir, patch_dir) return True - logger.info("apply patch {} failed".format(patch)) + logger.info("apply patch %s failed", patch) return False def apply_all_patches(self, *patches): @@ -164,7 +166,7 @@ class GiteeRepo(object): if patch in set(self._patch_files): rs.append(self.apply_patch(patch)) else: - logger.error("patch {} not exist".format(patch)) + logger.error("patch %s not exist", patch) rs.append(False) return 0 if all(rs) else (1 if any(rs) else -1) diff --git a/src/ac/common/linter.py b/src/ac/common/linter.py index 0b763037994ffaed1f1676cd02cb7dd0656bb398..95eacc86462c88abb4cc3aab2d688f155bd13833 100644 --- a/src/ac/common/linter.py +++ b/src/ac/common/linter.py @@ -73,13 +73,13 @@ class LinterCheck(object): * (E) error, for probable bugs in the code * (F) fatal, if an error occurred which prevented pylint from doing """ - logger.debug("check python file: {}".format(filepath)) + logger.debug("check python file: %s", filepath) # E0401: import module error pylint_cmd = "pylint3 --disable=E0401 {}".format(filepath) ret, out, _ = shell_cmd_live(pylint_cmd, cap_out=True, verbose=True) if ret: - logger.debug("pylint ret, {}".format(ret)) + logger.debug("pylint ret, %s", ret) return cls.get_summary_of_pylint(out) @@ -88,12 +88,12 @@ class LinterCheck(object): """ Check golang code by golint """ - logger.debug("check go file: {}".format(filepath)) + logger.debug("check go file: %s", filepath) golint_cmd = "golint {}".format(filepath) ret, out, _ = shell_cmd_live(golint_cmd, cap_out=True, verbose=True) if ret: - logger.debug("golint error, {}".format(ret)) + logger.debug("golint error, %s", ret) return {} return cls.get_summary_of_golint(out) @@ -103,13 +103,13 @@ class LinterCheck(object): """ Check c/c++ code by splint """ - logger.debug("check c/c++ file: {}".format(filepath)) + logger.debug("check c/c++ file: %s", filepath) splint_cmd = "splint {}".format(filepath) #ret, out, _ = shell_cmd_live(splint_cmd, cap_out=True, verbose=True) ret, out, _ = shell_cmd(splint_cmd) if ret: - logger.debug("splint error, {}".format(ret)) + logger.debug("splint error, %s", ret) return {} return cls.get_summary_of_splint(out) diff --git a/src/ac/common/pyrpm.py b/src/ac/common/pyrpm.py index 7068aa9156e5e5082646c70c4dcc44064f6d87b7..b958bb6fa77193aaadbdd1a912700f6c106d631c 100644 --- a/src/ac/common/pyrpm.py +++ b/src/ac/common/pyrpm.py @@ -174,7 +174,7 @@ _tags = [ _NameValue("group", re_tag_compile(r"^Group\s*:\s*(\S+)")), _NameValue("url", re_tag_compile(r"^URL\s*:\s*(\S+)")), _NameValue("buildroot", re_tag_compile(r"^BuildRoot\s*:\s*(\S+)")), - _NameValue("buildarch", re_tag_compile(r"^ExclusiveArch\s*:\s*(\S+)")), + _NameValue("buildarch", re_tag_compile(r"^ExclusiveArch\s*:\s*(\S.*)")), _ListAndDict("sources", re_tag_compile(r"^(Source\d*)\s*:\s*(\S+)")), _ListAndDict("patches", re_tag_compile(r"^(Patch\d*)\s*:\s*(\S+)")), _List("build_requires", re_tag_compile(r"^BuildRequires\s*:\s*(.+)")), diff --git a/src/ac/common/rpm_spec_adapter.py b/src/ac/common/rpm_spec_adapter.py index a714476a2d5254a961818f3839fa620525213c01..f32ddba59983b1eb8a497e48818741c7a7bff6d1 100644 --- a/src/ac/common/rpm_spec_adapter.py +++ b/src/ac/common/rpm_spec_adapter.py @@ -70,7 +70,7 @@ class RPMSpecAdapter(object): """ try: value = self.buildarch - logger.debug("build arch: {}".format(value)) + logger.debug("build arch: %s", value) if "x86_64" in value.lower(): return True @@ -85,7 +85,7 @@ class RPMSpecAdapter(object): """ try: value = self.buildarch - logger.debug("build arch: {}".format(value)) + logger.debug("build arch: %s", value) if "aarch64" in value.lower(): return True @@ -111,7 +111,7 @@ class RPMSpecAdapter(object): version_n = "{}{}".format(version_n, '.0' * (len(version_o.split('.')) - len(version_n.split('.')))) version_o = "{}{}".format(version_o, '.0' * (len(version_n.split('.')) - len(version_o.split('.')))) - logger.debug("compare versions: {} vs {}".format(version_n, version_o)) + logger.debug("compare versions: %s vs %s", version_n, version_o) z = zip(version_n.split("."), version_o.split(".")) for p in z: @@ -121,7 +121,7 @@ class RPMSpecAdapter(object): elif int(p[0]) > int(p[1]): return 1 except ValueError as exc: - logger.debug("check version exception, {}".format(exc)) + logger.debug("check version exception, %s", exc) continue return 0 diff --git a/src/ac/common/scanoss.py b/src/ac/common/scanoss.py index 39ce0391c02d47116c34cd0d9ba4b3533aa996df..2262702efe90fef902210868977c1030c2f259e4 100644 --- a/src/ac/common/scanoss.py +++ b/src/ac/common/scanoss.py @@ -50,7 +50,7 @@ class ScanOSS(object): try: json_format = json.loads(result) except ValueError: - logger.exception("illegal scanoss result, \"{}\"".format(result)) + logger.exception("illegal scanoss result, \"%s\"", result) return True snippets = 0 @@ -67,7 +67,7 @@ class ScanOSS(object): detail_trs.append(self.__class__.detail_trs(filename, item)) - logger.debug("snippets: {}, files: {}".format(snippets, files)) + logger.debug("snippets: %s, files: %s", snippets, files) detail = "{th}{trs}
".format( th=self.__class__.detail_th(), trs="\n".join(detail_trs)) @@ -161,7 +161,7 @@ class ScanOSS(object): :param directory: 需要扫描的目录 :return: """ - logger.debug("scan dir: {}".format(directory)) + logger.debug("scan dir: %s", directory) #scanoss_cmd = "scanner.py --format {} {} --apiurl {} {}".format( # "plain", "--key {}".format(self._key) if self._key else "", self._api_url, directory) scanoss_cmd = "scanner.py --blacklist {} --format {} {} --apiurl {} {}".format( @@ -169,8 +169,8 @@ class ScanOSS(object): ret, out, err = shell_cmd(scanoss_cmd) if ret: - logger.error("scanoss error, {}".format(ret)) - logger.error("{}".format(err)) + logger.error("scanoss error, %s", ret) + logger.error("%s", err) return True return self.result_analysis(out) diff --git a/src/ac/framework/ac.py b/src/ac/framework/ac.py index 9beda0be37b02d3113688f19281e695b52723cc7..f208b007df296d8c101eb01760b9d37d5ba2816f 100644 --- a/src/ac/framework/ac.py +++ b/src/ac/framework/ac.py @@ -49,7 +49,7 @@ class AC(object): self.load_check_elements_from_acl_directory(acl_path) self.load_check_elements_from_conf(conf, community) - logger.debug("check list: {}".format(self._ac_check_elements)) + logger.debug("check list: %s", self._ac_check_elements) @staticmethod def is_repo_support_check(repo, check_element): @@ -74,7 +74,7 @@ class AC(object): """ for element in self._ac_check_elements: check_element = self._ac_check_elements[element] - logger.debug("check {}".format(element)) + logger.debug("check %s", element) # show in gitee, must starts with "check_" hint = check_element.get("hint", "check_{}".format(element)) @@ -82,25 +82,25 @@ class AC(object): hint = "check_{}".format(hint) if not self.__class__.is_repo_support_check(repo, check_element): - logger.debug("{} not support check".format(repo)) + logger.debug("%s not support check", repo) continue # import module module_path = check_element.get("module", "{}.check_{}".format(element, element)) # eg: spec.check_spec try: module = importlib.import_module("." + module_path, self._acl_package) - logger.debug("load module {} succeed".format(module_path)) + logger.debug("load module %s succeed", module_path) except ImportError as exc: - logger.exception("import module {} exception, {}".format(module_path, exc)) + logger.exception("import module %s exception, %s", module_path, exc) continue # import entry entry_name = check_element.get("entry", "Check{}".format(element.capitalize())) try: entry = getattr(module, entry_name) - logger.debug("load entry \"{}\" succeed".format(entry_name)) + logger.debug("load entry \"%s\" succeed", entry_name) except AttributeError as exc: - logger.warning("entry \"{}\" not exist in module {}, {}".format(entry_name, module_path, exc)) + logger.warning("entry \"%s\" not exist in module %s, %s", entry_name, module_path, exc) continue # new a instance @@ -109,26 +109,26 @@ class AC(object): entry = entry(workspace, repo, check_element) # new a instance except Exception as exc: self._ac_check_result.append({"name": hint, "result": FAILED.val}) - logger.exception("new a instance of class {} exception, {}".format(entry_name, exc)) + logger.exception("new a instance of class %s exception, %s", entry_name, exc) continue if not callable(entry): # check callable - logger.warning("entry {} not callable".format(entry_name)) + logger.warning("entry %s not callable", entry_name) continue # do ac check try: result = entry(**kwargs) - logger.debug("check result {} {}".format(element, result)) + logger.debug("check result %s %s", element, result) except Exception as exc: - logger.exception("check exception, {} {}".format(element, exc)) + logger.exception("check exception, %s %s", element, exc) continue self._ac_check_result.append({"name": hint, "result": result.val}) dataset.set_attr("access_control.build.acl.{}".format(element), result.hint) dataset.set_attr("access_control.build.content", self._ac_check_result) - logger.debug("ac result: {}".format(self._ac_check_result)) + logger.debug("ac result: %s", self._ac_check_result) def load_check_elements_from_acl_directory(self, acl_dir): """ @@ -136,7 +136,7 @@ class AC(object): :return: """ for filename in os.listdir(acl_dir): - if os.path.isdir(os.path.join(acl_dir, filename)): + if filename != "__pycache__" and os.path.isdir(os.path.join(acl_dir, filename)): self._ac_check_elements[filename] = {} # don't worry, using default when checking def load_check_elements_from_conf(self, conf_file, community): @@ -150,18 +150,18 @@ class AC(object): with open(conf_file, "r") as f: content = yaml.safe_load(f) except IOError: - logger.exception("ac conf file {} not exist".format(conf_file)) + logger.exception("ac conf file %s not exist", conf_file) return except YAMLError: logger.exception("illegal conf file format") return elements = content.get(community, {}) - logger.debug("community \"{}\" conf: {}".format(community, elements)) + logger.debug("community \"%s\" conf: %s", community, elements) for name in elements: if name in self._ac_check_elements: if elements[name].get("exclude"): - logger.debug("exclude: {}".format(name)) + logger.debug("exclude: %s", name) self._ac_check_elements.pop(name) else: self._ac_check_elements[name] = elements[name] @@ -172,7 +172,7 @@ class AC(object): :param ac_file: :return: """ - logger.debug("save ac result to file {}".format(ac_file)) + logger.debug("save ac result to file %s", ac_file) with open(ac_file, "w") as f: f.write("ACL={}".format(json.dumps(self._ac_check_result))) @@ -202,12 +202,12 @@ def init_args(): parser.add_argument("-l", type=str, dest="trigger_link", help="job trigger link") # scanoss - parser.add_argument("--scanoss-api-key", type=str, dest="scanoss_api_key", help="scanoss api key") - parser.add_argument("--scanoss-api-url", type=str, dest="scanoss_api_url", - default="https://osskb.org/api/scan/direct", help="scanoss api url") parser.add_argument("--scanoss-output", type=str, dest="scanoss_output", default="scanoss_result", help="scanoss result output") - parser.add_argument("--scanoss-repo-path", type=str, dest="scanoss_repo", help="scanoss result repo path") + + parser.add_argument("--codecheck-api-key", type=str, dest="codecheck_api_key", help="codecheck api key") + parser.add_argument("--codecheck-api-url", type=str, dest="codecheck_api_url", + default="https://majun.osinfra.cn:8384/api/openlibing/codecheck", help="codecheck api url") return parser.parse_args() @@ -222,10 +222,10 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("ac") - logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git".format(args.community, args.repo)) + logger.info("using credential %s", args.account.split(":")[0]) + logger.info("cloning repository https://gitee.com/%s/%s.git ", args.community, args.repo) logger.info("clone depth 4") - logger.info("checking out pull request {}".format(args.pr)) + logger.info("checking out pull request %s", args.pr) # notify gitee from src.proxy.gitee_proxy import GiteeProxy @@ -254,7 +254,6 @@ if "__main__" == __name__: logging.getLogger("elasticsearch").setLevel(logging.WARNING) logging.getLogger("kafka").setLevel(logging.WARNING) - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) # download repo @@ -266,9 +265,7 @@ if "__main__" == __name__: dd.set_attr_etime("access_control.scm.etime") dd.set_attr_etime("access_control.job.etime") - #dd.set_attr("access_control.job.result", "successful") - ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) - kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) + kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) logger.info("fetch finished -") sys.exit(-1) else: @@ -289,16 +286,18 @@ if "__main__" == __name__: gp.create_tags_of_pr(args.pr, "ci_processing") # scanoss conf - scanoss = {"api_key": args.scanoss_api_key, "api_url": args.scanoss_api_url, - "output": args.scanoss_output, "repo_path": args.scanoss_repo} + scanoss = {"output": args.scanoss_output} + + codecheck = {"pr_url": "https://gitee.com/{}/{}/pulls/{}".format(args.community, args.repo, args.pr), + "pr_number": args.pr, "codecheck_api_url": args.codecheck_api_url, "codecheck_api_key": args.codecheck_api_key + } # build ac = AC(os.path.join(os.path.dirname(os.path.realpath(__file__)), "ac.yaml"), args.community) - ac.check_all(workspace=args.workspace, repo=args.repo, dataset=dd, tbranch=args.tbranch, scanoss=scanoss) + ac.check_all(workspace=args.workspace, repo=args.repo, dataset=dd, tbranch=args.tbranch, scanoss=scanoss, + codecheck=codecheck) dd.set_attr_etime("access_control.build.etime") ac.save(args.output) dd.set_attr_etime("access_control.job.etime") - #dd.set_attr("access_control.job.result", "successful") - ep.insert(index="openeuler_statewall_ac", body=dd.to_dict()) - kp.send("openeuler_statewall_ci_ac", value=dd.to_dict()) + kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 3a12e87311ed086775cd34f6505adaeb4a8ece95..5cabb6fe194b0abda1bc08f8bbf7676d23a1c5ba 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -21,6 +21,8 @@ src-openeuler: entry: CheckLicense sca: exclude: True + openlibing: + exclude: True openeuler: spec: exclude: True @@ -34,4 +36,9 @@ openeuler: hint: check_sca module: sca.check_sca entry: CheckSCA - allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform"] + allow_list: ["openeuler-jenkins", "pkgship", "stratovirt", "secGear", "isula-transform", "kunpengsecl", "release-tools"] + openlibing: + hint: code + module: openlibing.check_code + entry: CheckCode + allow_list: ["pkgship", "kunpengsecl", "release-tools"] diff --git a/src/ac/framework/ac_base.py b/src/ac/framework/ac_base.py index 7deb3b54b926831c75bf0b8c5ffd39b66f9041e5..9f3e9d73f3b85ed74582244f24c3086b36aac64d 100644 --- a/src/ac/framework/ac_base.py +++ b/src/ac/framework/ac_base.py @@ -57,27 +57,27 @@ class BaseCheck(object): result = SUCCESS for name in items: try: - logger.debug("check {}".format(name)) + logger.debug("check %s", name) method = getattr(self, "check_{}".format(name)) rs = method() - logger.debug("{} -> {}".format(name, rs)) + logger.debug("%s -> %s", name, rs) except Exception as e: # 忽略代码错误 - logger.exception("internal error: {}".format(e)) + logger.exception("internal error: %s", e) continue ignored = True if self._conf and name in self._conf.get("ignored", []) else False - logger.debug("{} ignore: {}".format(name, ignored)) + logger.debug("%s ignore: %s", name, ignored) if rs is SUCCESS: - logger.info("check {:<30}pass".format(name)) + logger.info("check %s pass", name) elif rs is WARNING: - logger.warning("check {:<30}warning{}".format(name, " [ignored]" if ignored else "")) + logger.warning("check %s warning %s", name, " [ignored]" if ignored else "") elif rs is FAILED: - logger.error("check {:<30}fail{}".format(name, " [ignored]" if ignored else "")) + logger.error("check %s fail %s", name, " [ignored]" if ignored else "") else: # never here - logger.exception("check {:<30}exception{}".format(name, " [ignored]" if ignored else "")) + logger.exception("check %s exception %s", name, " [ignored]" if ignored else "") continue if not ignored: @@ -91,6 +91,6 @@ class BaseCheck(object): """ members = inspect.getmembers(self, inspect.ismethod) items = [member[0].replace("check_", "") for member in members if member[0].startswith("check_")] - logger.debug("check items: {}".format(items)) + logger.debug("check items: %s", items) return self.start_check_with_order(*items) diff --git a/src/build/build_rpm_package.py b/src/build/build_rpm_package.py index e33d2e1dde4c73811475129b3b0c54ad2690704c..3f2d845c2bdbd6f09d0598ec6d20d67b233585f2 100755 --- a/src/build/build_rpm_package.py +++ b/src/build/build_rpm_package.py @@ -180,3 +180,24 @@ class BuildRPMPackage(object): for filename in filenames: name = self.extract_rpm_name(filename) self._rpm_packages["srpm"][name] = {"name": name, "fullname": filename} + + def iter_all_rpm(self): + """ + 遍历所有rpm包，返回包在local的路径 + :return: + """ + packages = self._rpm_packages.get("rpm", {}) + for name in packages: + package = packages[name] + yield name, os.path.join(self._rpmbuild_dir, "RPMS", package["arch"], package["fullname"]) + + def iter_all_srpm(self): + """ + 遍历所有source rpm包，返回包在local的路径 + :return: + """ + packages = self._rpm_packages.get("rpm", {}) + + for name in packages: + package = packages[name] + yield name, os.path.join(self._rpmbuild_dir, "SRPMS", package["fullname"]) diff --git a/src/build/extra_work.py b/src/build/extra_work.py index 39f8e906b3fc0227f2c1e933db02afacecdc159d..149b701fc32767743939f4b639d2b0f5b4838c0b 100755 --- a/src/build/extra_work.py +++ b/src/build/extra_work.py @@ -22,6 +22,8 @@ import logging.config import logging import yaml +from src.build.obs_repo_source import OBSRepoSource + class ExtraWork(object): """ @@ -47,7 +49,7 @@ class ExtraWork(object): try: with open(pkgship_meta_path, "r") as f: pkgship_meta = yaml.safe_load(f) - logger.debug("pkgship meta: {}".format(pkgship_meta)) + logger.debug("pkgship meta: %s", pkgship_meta) if pkgship_meta.get("compare_version") == 1: # version upgrade logger.debug("pkgship: notify") return True @@ -91,8 +93,8 @@ class ExtraWork(object): #get rpms curr_rpm = self._rpm_package.main_package_local() last_rpm = self._rpm_package.last_main_package(package_arch, package_url) - logger.debug("curr_rpm: {}".format(curr_rpm)) - logger.debug("last_rpm: {}".format(last_rpm)) + logger.debug("curr_rpm: %s", curr_rpm) + logger.debug("last_rpm: %s", last_rpm) if not curr_rpm or not last_rpm: logger.info("no rpms") return @@ -106,8 +108,8 @@ class ExtraWork(object): debuginfos = None curr_rpm_debug = self._rpm_package.debuginfo_package_local() last_rpm_debug = self._rpm_package.last_debuginfo_package(package_arch, package_url) - logger.debug("curr_rpm_debug: {}".format(curr_rpm_debug)) - logger.debug("last_rpm_debug: {}".format(last_rpm_debug)) + logger.debug("curr_rpm_debug: %s", curr_rpm_debug) + logger.debug("last_rpm_debug: %s", last_rpm_debug) if curr_rpm_debug and last_rpm_debug: debuginfos = [last_rpm_debug, curr_rpm_debug] @@ -121,9 +123,9 @@ class ExtraWork(object): check_abi = CheckAbi(result_output_file=output, input_rpms_path=related_rpms_url) ret = check_abi.process_with_rpm(rpms, debuginfos) if ret == 1: - logger.error("check abi error: {}".format(ret)) + logger.error("check abi error: %s", ret) else: - logger.debug("check abi ok: {}".format(ret)) + logger.debug("check abi ok: %s", ret) if os.path.exists(output): # change of abi @@ -133,7 +135,7 @@ class ExtraWork(object): else: comment = {"name": "check_abi/{}/{}".format(package_arch, self._repo), "result": "SUCCESS"} - logger.debug("check abi comment: {}".format(comment)) + logger.debug("check abi comment: %s", comment) try: with open(comment_file, "r") as f: # one repo with multi build package comments = yaml.safe_load(f) @@ -149,17 +151,64 @@ class ExtraWork(object): logger.exception("yaml load check abi comment file exception") comments.append(comment) - logger.debug("check abi comments: {}".format(comments)) + logger.debug("check abi comments: %s", comments) try: with open(comment_file, "w") as f: yaml.safe_dump(comments, f) # list except IOError: logger.exception("save check abi comment exception") + def check_install_rpm(self, branch_name, arch, install_root): + """ + 检查生成的rpm是否可以安装 + :param branch_name: 分支名 + :param arch: cpu架构 + :param install_root: 安装根路径 + :return: + """ + logger.info("*** start check install start ***") + + # 1. prepare install root directory + _ = not os.path.exists(install_root) and os.mkdir(install_root) + logger.info("create install root directory: %s", install_root) + + repo_name_prefix = "check_install" + + # 2. prepare repo + repo_source = OBSRepoSource("http://119.3.219.20:82") # obs 实时构建repo地址 + repo_config = repo_source.generate_repo_info(branch_name, arch, "check_install") + logger.info("repo source config:\n%s", repo_config) + + # write to /etc/yum.repos.d + with open("obs_realtime.repo", "w+") as f: + f.write(repo_config) + + # 3. dnf install using repo name start with "check_install" + names = [] + packages = [] + for name, package in self._rpm_package.iter_all_rpm(): + # ignore debuginfo rpm + if "debuginfo" in name or "debugsource" in name: + logger.debug("ignore debug rpm: %s", name) + continue + names.append(name) + packages.append(package) + + logger.info("install rpms: %s", names) + if packages: + check_install_cmd = "sudo dnf install -y --installroot={} --setopt=reposdir=. {}".format( + install_root, " ".join(packages)) + ret, _, err = shell_cmd_live(check_install_cmd, verbose=True) + if ret: + logger.error("install rpms error, %s, %s", ret, err) + else: + logger.info("install rpm success") + + if "__main__" == __name__: args = argparse.ArgumentParser() - args.add_argument("-f", type=str, dest="func", choices=("notify", "checkabi"), help="function") + args.add_argument("-f", type=str, dest="func", choices=("notify", "checkabi", "checkinstall"), help="function") args.add_argument("-p", type=str, dest="package", help="obs package") args.add_argument("-a", type=str, dest="arch", help="build arch") @@ -180,6 +229,8 @@ if "__main__" == __name__: args.add_argument("-b", type=str, dest="obs_repo_url", help="obs repo where rpm saved") args.add_argument("-s", type=str, dest="obs_addr", help="obs address") args.add_argument("-r", type=str, dest="branch_name", help="obs project name") + + args.add_argument("--install-root", type=str, dest="install_root", help="check install root dir") args = args.parse_args() _ = not os.path.exists("log") and os.mkdir("log") @@ -203,3 +254,5 @@ if "__main__" == __name__: # run before copy rpm to rpm repo ew.check_rpm_abi(args.rpm_repo_url, args.arch, args.output, args.committer, args.comment_file, args.obs_addr, args.branch_name, args.obs_repo_url) + elif args.func == "checkinstall": + ew.check_install_rpm(args.branch_name, args.arch, args.install_root) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 56ef4c09374fa97801f6fbaebcfed82061359eb1..5d778bca4c7cfb32e400170fadd1fb6b3b54e00e 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -82,7 +82,7 @@ class Comment(object): base_job_name = os.environ.get("JOB_NAME") base_build_id = os.environ.get("BUILD_ID") base_build_id = int(base_build_id) - logger.debug("base_job_name: {}, base_build_id: {}".format(base_job_name, base_build_id)) + logger.debug("base_job_name: %s, base_build_id: %s", base_job_name, base_build_id) base_build = jenkins_proxy.get_build(base_job_name, base_build_id) logger.debug("get base build") self._up_builds = jenkins_proxy.get_upstream_builds(base_build) @@ -120,7 +120,7 @@ class Comment(object): try: acl = json.loads(os.environ["ACL"]) - logger.debug("ac result: {}".format(acl)) + logger.debug("ac result: %s", acl) except ValueError: logger.exception("invalid ac result format") return [] @@ -138,7 +138,7 @@ class Comment(object): comments.append(self.__class__.comment_html_table_tr_rowspan( item["name"], ac_result.emoji, ac_result.hint)) - logger.info("ac comment: {}".format(comments)) + logger.info("ac comment: %s", comments) return comments @@ -158,7 +158,7 @@ class Comment(object): comments.append(self.__class__.comment_html_table_tr( name, ac_result.emoji, ac_result.hint, "{}{}".format(build_url, "console"), build.buildno)) - logger.info("build comment: {}".format(comments)) + logger.info("build comment: %s", comments) return comments @@ -178,25 +178,25 @@ class Comment(object): return False for check_abi_comment_file in self._check_abi_comment_files: - logger.debug("check abi comment file: {}".format(check_abi_comment_file)) + logger.debug("check abi comment file: %s", check_abi_comment_file) if not os.path.exists(check_abi_comment_file): # check abi评论文件存在 continue for build in builds: name = build.job._data["fullName"] - logger.debug("check build {}".format(name)) + logger.debug("check build %s", name) if not match(name, check_abi_comment_file): # 找到匹配的jenkins build continue - logger.debug("build \"{}\" match".format(name)) + logger.debug("build \"%s\" match", name) status = build.get_status() - logger.debug("build state: {}".format(status)) + logger.debug("build state: %s", status) if ACResult.get_instance(status) == SUCCESS: # 保证build状态成功 with open(check_abi_comment_file, "r") as f: try: content = yaml.safe_load(f) except YAMLError: # yaml base exception logger.exception("illegal yaml format of check abi comment file ") - logger.debug("comment: {}".format(content)) + logger.debug("comment: %s", content) for item in content: ac_result = ACResult.get_instance(item.get("result")) comments.append(self.__class__.comment_html_table_tr( @@ -204,7 +204,7 @@ class Comment(object): "markdown" if "link" in item else "", hashtag=False)) break - logger.info("check abi comment: {}".format(comments)) + logger.info("check abi comment: %s", comments) return comments @@ -315,9 +315,7 @@ if "__main__" == __name__: logging.getLogger("kafka").setLevel(logging.WARNING) # upload to es - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.comment = params.comment", "params": dd.to_dict()} - #ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) diff --git a/src/build/obs_repo_source.py b/src/build/obs_repo_source.py new file mode 100644 index 0000000000000000000000000000000000000000..638e44c1f69c9c8d7d2b570d6c72665c0bd33a51 --- /dev/null +++ b/src/build/obs_repo_source.py @@ -0,0 +1,103 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-05-27 +# Description: obs repo as dnf source +# ********************************************************************************** +""" +from src.proxy.requests_proxy import do_requests +import logging + +logger = logging.getLogger("common") + + +class OBSRepoSource(object): + """ + 生成obs实时仓作为rpm源的配置 + """ + def __init__(self, repo_host): + """ + + :param repo_host: obs仓库host + """ + self._current_repo_host = repo_host + + @staticmethod + def repo_format(repo_id, repo_name, repo_baseurl): + """ + repo内容格式 + :param repo_id: + :param repo_name: + :param repo_baseurl: + :return: + """ + return "[{}]\nname={}\nbaseurl={}\nenabled=1\ngpgcheck=0\n".format(repo_id, repo_name, repo_baseurl) + + def generate_repo_info(self, branch, arch, repo_name_prefix): + """ + 不同的分支生成不同的repo + :param branch: + :param arch: + :param repo_name_prefix: + :return: + """ + repo_config = "" + + if branch == "master": + obs_path_part = "openEuler:/Mainline" + elif "openstack" in branch: + branch = branch.replace("_oe-", "_openEuler-") # openEuler abbr. + vendor, openstack, os = branch.split("_") + obs_path_part = (":/").join( + [os.replace("-", ":/"), vendor.replace("-", ":/"), openstack.replace("-", ":/")]) + obs_path_part_common = (":/").join( + [os.replace("-", ":/"), vendor.replace("-", ":/"), "openstack:/common"]) + obs_path_part_base = (":/").join([os.replace("-", ":/")]) + + # openstack need common and base + if "openstack-common" != openstack: + # openstack common + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part_common, arch) + if do_requests("GET", url) == 0: + logger.debug("add openstack common repo: %s", url) + repo_config += self.repo_format("openstack_common", repo_name_prefix + "_openstack_common", url) + + # openstack base + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part_base, arch) + if do_requests("GET", url) == 0: + logger.debug("add openstack base repo: %s", url) + repo_config += self.repo_format("openstack_base", repo_name_prefix + "_openstack_base", url) + else: + obs_path_part = branch.replace("-", ":/") + + logger.debug("branch=%s, obs_path_part=%s", branch, obs_path_part) + + # main + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part, arch) + if do_requests("GET", url) == 0: + logger.debug("add main repo: %s", url) + repo_config += self.repo_format(repo_name_prefix + "_main", repo_name_prefix + "_main", url) + + # epol + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part + ":/Epol", arch) + if do_requests("GET", url) == 0: + logger.debug("add epol repo: %s", url) + repo_config += self.repo_format(repo_name_prefix + "_epol", repo_name_prefix + "_epol", url) + + # extras + url = "{}/{}/standard_{}".format(self._current_repo_host, obs_path_part + ":/Extras", arch) + if do_requests("GET", url) == 0: + logger.debug("add extras repo: %s", url) + repo_config += self.repo_format(repo_name_prefix + "_extras", repo_name_prefix + "_extras", url)\ + + return repo_config diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 98cccaa8270d9d0f1044d9df6f6b7d938ea668f3..c9b4d9bc2447d64b7b39ca97a9721c3a9fa53e9f 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -33,22 +33,29 @@ class SinglePackageBuild(object): "openEuler-20.03-LTS": ["openEuler:20.03:LTS"], "openEuler-20.03-LTS-Next": ["openEuler:20.03:LTS:Next", "openEuler:20.03:LTS:Next:Epol"], "openEuler-EPOL-LTS": ["bringInRely"], - "openEuler-20.09": ["openEuler:20.09", "openEuler:20.09:Epol"], + "openEuler-20.09": ["openEuler:20.09", "openEuler:20.09:Epol", "openEuler:20.09:Extras"], "mkopeneuler-20.03": ["openEuler:Extras"], "openEuler-20.03-LTS-SP1": ["openEuler:20.03:LTS:SP1", "openEuler:20.03:LTS:SP1:Epol", "openEuler:20.03:LTS:SP1:Extras"], "openEuler-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2", "openEuler:20.03:LTS:SP2:Epol", "openEuler:20.03:LTS:SP2:Extras"], "openEuler-21.03": ["openEuler:21.03", "openEuler:21.03:Epol", "openEuler:21.03:Extras"], + "openEuler-21.09": ["openEuler:21.09", "openEuler:21.09:Epol", "openEuler:21.09:Extras"], "oepkg_openstack-common_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:common"], "oepkg_openstack-queens_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:queens"], "oepkg_openstack-rocky_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:rocky"], "oepkg_openstack-common_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:common"], "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"], - "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"] + "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"], + "oepkg_openstack-common_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:common"], + "oepkg_openstack-queens_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:queens"], + "oepkg_openstack-rocky_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:rocky"], + "openEuler-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3", "openEuler:20.03:LTS:SP3:Epol"], + "openEuler-22.03-LTS-Next": ["openEuler:22.03:LTS:Next", "openEuler:22.03:LTS:Next:Epol"] } BUILD_IGNORED_GITEE_BRANCH = ["riscv"] + PACKAGES_USE_ROOT = ["iproute", "libaio", "A-Ops", "multipath-tools"] def __init__(self, package, arch, target_branch): """ @@ -66,13 +73,14 @@ class SinglePackageBuild(object): 需要构建obs repo列表 :return: list """ - return OBSProxy.list_repos_of_arch(project, self._package, self._arch) + return OBSProxy.list_repos_of_arch(project, self._package, self._arch, show_exclude=True) - def build_obs_repos(self, project, repos, work_dir, code_dir): + def build_obs_repos(self, project, repos, spec, work_dir, code_dir): """ build :param project: 项目名 :param repos: obs repo + :param spec: 指定spec文件 :param code_dir: 码云代码在本地路径 :param work_dir: :return: @@ -97,11 +105,17 @@ class SinglePackageBuild(object): # osc build for repo in repos: - if not OBSProxy.build_package(project, self._package, repo["repo"], self._arch): - logger.error("build {} ... failed".format(repo["repo"])) + if repo["state"] == "excluded" and repo["mpac"] == "raspberrypi-kernel": + logger.info("repo %s:%s excluded", repo["repo"], repo["mpac"]) + continue + root_build = repo["mpac"] in self.PACKAGES_USE_ROOT + if not OBSProxy.build_package( + project, self._package, repo["repo"], self._arch, spec, repo["mpac"], + root_build=root_build, disable_cpio=True): + logger.error("build %s ... failed", repo["repo"]) return 3 - logger.info("build {} ... ok".format(repo["repo"])) + logger.info("build %s ... ok", repo["repo"]) logger.debug("build all repos ... finished") @@ -179,42 +193,50 @@ class SinglePackageBuild(object): ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: - logger.error("prepare build environ error, {}".format(ret)) + logger.error("prepare build environ error, %s", ret) return False return True - def build(self, work_dir, code_dir): + def build(self, spec, work_dir, code_dir): """ 入口 + :param spec: 指定spec文件 :param work_dir: obs工作目录 :param code_dir: 代码目录 :return: """ if self._branch in self.BUILD_IGNORED_GITEE_BRANCH: - logger.error("branch \"{}\" ignored".format(self._branch)) + logger.error("branch \"%s\" ignored", self._branch) return 0 if self._branch not in self.GITEE_BRANCH_PROJECT_MAPPING: - logger.error("branch \"{}\" not support yet".format(self._branch)) + logger.error("branch \"%s\" not support yet", self._branch) return 1 + has_any_repo_build = False for project in self.GITEE_BRANCH_PROJECT_MAPPING.get(self._branch): - logger.debug("start build project {}".format(project)) + logger.debug("start build project %s", project) obs_repos = self.get_need_build_obs_repos(project) if not obs_repos: - logger.info("all repos ignored of project {}".format(project)) + logger.info("all repos ignored of project %s", project) continue - logger.debug("build obs repos: {}".format(obs_repos)) - ret = self.build_obs_repos(project, obs_repos, work_dir, code_dir) + logger.debug("build obs repos: %s", obs_repos) + has_any_repo_build = True + ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) if ret > 0: - logger.debug("build run return {}".format(ret)) - logger.error("build {} {} {} ... {}".format(project, self._package, self._arch, "failed")) + logger.debug("build run return %s", ret) + logger.error("build %s %s %s ... %s", project, self._package, self._arch, "failed") return 1 # finish if any error else: - logger.info("build {} {} {} ... {}".format(project, self._package, self._arch, "ok")) + logger.info("build %s %s %s ... %s", project, self._package, self._arch, "ok") + + # if no repo build, regard as fail + if not has_any_repo_build: + logger.error("package not in any obs projects, please add package into obs") + return 1 return 0 @@ -238,6 +260,7 @@ def init_args(): parser.add_argument("-t", type=str, dest="account", help="gitee account") parser.add_argument("-o", type=str, dest="owner", default="src-openeuler", help="gitee owner") + parser.add_argument("--spec", type=str, dest="spec", default="", help="spec files") return parser.parse_args() @@ -250,10 +273,10 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") - logger.info("using credential {}".format(args.account.split(":")[0])) - logger.info("cloning repository https://gitee.com/{}/{}.git".format(args.owner, args.repo)) + logger.info("using credential %s", args.account.split(":")[0]) + logger.info("cloning repository https://gitee.com/%s/%s.git ", args.owner, args.repo) logger.info("clone depth 1") - logger.info("checking out pull request {}".format(args.pr)) + logger.info("checking out pull request %s", args.pr) from src.utils.dist_dataset import DistDataset from src.proxy.git_proxy import GitProxy @@ -272,7 +295,6 @@ if "__main__" == __name__: logging.getLogger("elasticsearch").setLevel(logging.WARNING) logging.getLogger("kafka").setLevel(logging.WARNING) - ep = ESProxy(os.environ["ESUSERNAME"], os.environ["ESPASSWD"], os.environ["ESURL"], verify_certs=False) kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) # download repo @@ -291,7 +313,6 @@ if "__main__" == __name__: query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) sys.exit(-1) else: @@ -302,7 +323,7 @@ if "__main__" == __name__: dd.set_attr_stime("spb.build.stime") spb = SinglePackageBuild(args.package, args.arch, args.branch) - rs = spb.build(args.workspace, args.code) + rs = spb.build(args.spec, args.workspace, args.code) dd.set_attr("spb.build.result", "failed" if rs else "successful") dd.set_attr_etime("spb.build.etime") @@ -311,6 +332,5 @@ if "__main__" == __name__: # upload to es query = {"term": {"id": args.comment_id}} script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - ep.update_by_query(index="openeuler_statewall_ac", query=query, script=script) kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) sys.exit(rs) diff --git a/src/build/related_rpm_package.py b/src/build/related_rpm_package.py index 97b891cacf3d275b6a728307d9c0add01573b676..e07e5a5c0a934855885089d7f9379ad9fb57cddd 100755 --- a/src/build/related_rpm_package.py +++ b/src/build/related_rpm_package.py @@ -44,7 +44,10 @@ class RelatedRpms(object): "oepkg_openstack-queens_oe-20.03-LTS-SP2": ["openEuler:20.03:LTS:SP2:oepkg:openstack:queens"], "oepkg_openstack-common_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:common"], "oepkg_openstack-rocky_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:rocky"], - "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"] + "oepkg_openstack-queens_oe-20.03-LTS-Next": ["openEuler:20.03:LTS:Next:oepkg:openstack:queens"], + "oepkg_openstack-common_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:common"], + "oepkg_openstack-rocky_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:rocky"], + "oepkg_openstack-queens_oe-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3:oepkg:openstack:queens"] } def __init__(self, obs_addr, obs_repo_url, branch_name, package_arch): diff --git a/src/conf/deny_list.sbom b/src/conf/deny_list.sbom index 87e0d67491cc81ad5fb29dd299270af215f3951e..7f1e3ff1399dfabfc4fcd7183c5564c2d470c437 100644 --- a/src/conf/deny_list.sbom +++ b/src/conf/deny_list.sbom @@ -2,24 +2,316 @@ "comment": "Component Denylist", "components": [ { - "publish": "openeuler", + "publisher": "openeuler", "name": "pkgship" }, { - "publish": "openeuler", + "publisher": "openeuler-mirror", + "name": "pkgship" + }, + { + "publisher": "src-openeuler", + "name": "pkgship" + }, + { + "publisher": "openeuler", "name": "openeuler-jenkins" }, { - "publish": "openeuler", + "publisher": "openeuler-mirror", + "name": "openeuler-jenkins" + }, + { + "publisher": "src-openeuler", + "name": "openeuler-jenkins" + }, + { + "publisher": "openeuler", + "name": "stratovirt" + }, + { + "publisher": "openeuler-mirror", "name": "stratovirt" }, { - "publish": "openeuler", + "publisher": "src-openeuler", + "name": "stratovirt" + }, + { + "publisher": "openeuler", + "name": "secGear" + }, + { + "publisher": "openeuler-mirror", "name": "secGear" }, { - "publish": "openeuler", + "publisher": "src-openeuler", + "name": "secGear" + }, + { + "publisher": "openeuler", + "name": "isula-transform" + }, + { + "publisher": "openeuler-mirror", + "name": "isula-transform" + }, + { + "publisher": "src-openeuler", "name": "isula-transform" - } + }, + { + "publisher": "openeuler", + "name": "kernel" + }, + { + "publisher": "openeuler-mirror", + "name": "kernel" + }, + { + "publisher": "src-openeuler", + "name": "kernel" + }, + { + "publisher": "openeuler", + "name": "async-libfuse" + }, + { + "publisher": "openeuler-mirror", + "name": "async-libfuse" + }, + { + "publisher": "src-openeuler", + "name": "async-libfuse" + }, + { + "publisher": "openeuler", + "name": "authz" + }, + { + "publisher": "openeuler-mirror", + "name": "authz" + }, + { + "publisher": "src-openeuler", + "name": "authz" + }, + { + "publisher": "openeuler", + "name": "clibcni" + }, + { + "publisher": "openeuler-mirror", + "name": "clibcni" + }, + { + "publisher": "src-openeuler", + "name": "clibcni" + }, + { + "publisher": "openeuler", + "name": "isula-build" + }, + { + "publisher": "openeuler-mirror", + "name": "isula-build" + }, + { + "publisher": "src-openeuler", + "name": "isula-build" + }, + { + "publisher": "openeuler", + "name": "iSulad" + }, + { + "publisher": "openeuler-mirror", + "name": "iSulad" + }, + { + "publisher": "src-openeuler", + "name": "iSulad" + }, + { + "publisher": "openeuler", + "name": "iSulad-img" + }, + { + "publisher": "openeuler-mirror", + "name": "iSulad-img" + }, + { + "publisher": "src-openeuler", + "name": "iSulad-img" + }, + { + "publisher": "openeuler", + "name": "iSula-libutils" + }, + { + "publisher": "openeuler-mirror", + "name": "iSula-libutils" + }, + { + "publisher": "src-openeuler", + "name": "iSula-libutils" + }, + { + "publisher": "openeuler", + "name": "isula-transform" + }, + { + "publisher": "openeuler-mirror", + "name": "isula-transform" + }, + { + "publisher": "src-openeuler", + "name": "isula-transform" + }, + { + "publisher": "openeuler", + "name": "lcr" + }, + { + "publisher": "openeuler-mirror", + "name": "lcr" + }, + { + "publisher": "src-openeuler", + "name": "lcr" + }, + { + "publisher": "openeuler", + "name": "lib-shim-v2" + }, + { + "publisher": "openeuler-mirror", + "name": "lib-shim-v2" + }, + { + "publisher": "src-openeuler", + "name": "lib-shim-v2" + }, + { + "publisher": "openeuler", + "name": "lxcfs-tools" + }, + { + "publisher": "openeuler-mirror", + "name": "lxcfs-tools" + }, + { + "publisher": "src-openeuler", + "name": "lxcfs-tools" + }, + { + "publisher": "openeuler", + "name": "syscontainer-tools" + }, + { + "publisher": "openeuler-mirror", + "name": "syscontainer-tools" + }, + { + "publisher": "src-openeuler", + "name": "syscontainer-tools" + }, + { + "publisher": "openeuler", + "name": "A-Tune" + }, + { + "publisher": "openeuler-mirror", + "name": "A-Tune" + }, + { + "publisher": "src-openeuler", + "name": "A-Tune" + }, + { + "publisher": "openeuler", + "name": "A-Tune-Collector" + }, + { + "publisher": "openeuler-mirror", + "name": "A-Tune-Collector" + }, + { + "publisher": "src-openeuler", + "name": "A-Tune-Collector" + }, + { + "publisher": "openeuler", + "name": "A-Tune-UI" + }, + { + "publisher": "openeuler-mirror", + "name": "A-Tune-UI" + }, + { + "publisher": "src-openeuler", + "name": "A-Tune-UI" + }, + { + "publisher": "openeuler", + "name": "prefetch_tuning" + }, + { + "publisher": "openeuler-mirror", + "name": "prefetch_tuning" + }, + { + "publisher": "src-openeuler", + "name": "prefetch_tuning" + }, + { + "publisher": "openeuler", + "name": "wisdom-advisor" + }, + { + "publisher": "openeuler-mirror", + "name": "wisdom-advisor" + }, + { + "publisher": "src-openeuler", + "name": "wisdom-advisor" + }, + { + "publisher": "openeuler", + "name": "qemu" + }, + { + "publisher": "openeuler-mirror", + "name": "qemu" + }, + { + "publisher": "src-openeuler", + "name": "qemu" + }, + { + "publisher": "openeuler", + "name": "vmtop" + }, + { + "publisher": "openeuler-mirror", + "name": "vmtop" + }, + { + "publisher": "src-openeuler", + "name": "vmtop" + }, + { + "publisher": "openeuler", + "name": "kunpengsecl" + }, + { + "ipublisher": "openeuler-mirror", + "name": "kunpengsecl" + }, + { + "publisher": "src-openeuler", + "name": "kunpengsecl" + } ] } diff --git a/src/dockerfile/README.md b/src/dockerfile/README.md new file mode 100644 index 0000000000000000000000000000000000000000..622bee845bdf37535e3563f8d609184b598c8d04 --- /dev/null +++ b/src/dockerfile/README.md @@ -0,0 +1,36 @@ +# docker images +## openeuler +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:20.03-lts + +## openjdk 11-jdk-stretch based on openeuler +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/11-jdk-stretch:20.03-lts + +> Dockerfile: openjdk-openeuler + +## openeuler ci image based on openjdk +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:21.03 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.09 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.03-lts-sp1 +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/common:20.03-lts + +> Dockerfile: ci-common + +## src-openeuler ci image based on openjdk +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/soe:base + +> Dockerfile: ci-soe-base + +> choose an openjdk image that from stable openeuler version + +- swr.cn-north-4.myhuaweicloud.com/openeuler/ci/soe:{version} + +## openeuler image based on openeuler-sp2 +> install the python packages required by release-tools based on sp2 +> Dockerfile: release-tools-dockerfile +- docker image name: openeuler-sp2-release-tools \ No newline at end of file diff --git a/src/dockerfile/obs-base b/src/dockerfile/ci-base-ubuntu similarity index 100% rename from src/dockerfile/obs-base rename to src/dockerfile/ci-base-ubuntu diff --git a/src/dockerfile/ci-bisheng-riscv b/src/dockerfile/ci-bisheng-riscv new file mode 100644 index 0000000000000000000000000000000000000000..b4169bbd874008fba4fe043443904a78e1b74852 --- /dev/null +++ b/src/dockerfile/ci-bisheng-riscv @@ -0,0 +1,106 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +# Install qemu 5.2.0 +RUN set -eux; \ + yum update; yum clean all; \ + yum -y groupinstall "Development Tools"; \ + yum -y install pixman-devel zlib-* glib2 glib2-devel perl tar xz zip unzip wget; \ + cd /usr/src; \ + wget -q http://121.36.53.23/bisheng_riscv/ninja-linux.zip; \ + unzip ninja-linux.zip; \ + cp ninja /usr/bin; \ + wget -q http://121.36.53.23/bisheng_riscv/qemu-5.2.0.tar.xz; \ + tar xJf qemu-5.2.0.tar.xz; \ + cd qemu-5.2.0 && mkdir build && cd build; \ + ../configure --target-list=riscv64-linux-user,riscv64-softmmu; \ + make install -j; \ + rm -rf /usr/src/qemu-5.2.0* /usr/src/ninja*; + +# RISCV toolchain +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/riscv_toolchain_linux64.tar.xz; \ + tar xJf riscv_toolchain_linux64.tar.xz; \ + rm riscv_toolchain_linux64.tar.xz; + +# fedora sysroot +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/fedora28_riscv_smlroot.tar.xz; \ + tar xJf fedora28_riscv_smlroot.tar.xz; \ + ln -s /opt/fedora28_riscv_root/usr/lib64/ld-linux-riscv64-lp64d.so.1 /lib/ld-linux-riscv64-lp64d.so.1; \ + rm fedora28_riscv_smlroot.tar.xz; + +# Packages for openjdk build +RUN set -eux; \ + yum -y install java-1.8.0-openjdk-devel cups-devel libXtst-devel libXt-devel libXrender-devel libXi-devel alsa-lib-devel libffi-devel elfutils-libelf-devel libstdc++-static libatomic + +# Test tool packages +RUN set -eux; \ + yum -y install gcc gcc-c++ unzip zlib-devel perl-devel libcurl-devel openssl-devel mercurial perl perl-Test-Simple xorg-x11-server-Xorg xorg-x11-server-Xvfb + +# JTREG +RUN cd /opt; \ + wget -q http://121.36.53.23/bisheng_riscv/jtreg-4.2.0-tip.tar.gz; \ + mkdir jtreg-4; tar -xf jtreg-4.2.0-tip.tar.gz -C jtreg-4 --strip-components 1; \ + wget -q http://121.36.53.23/bisheng_riscv/jtreg-5.1-b01.tar.gz; \ + mkdir jtreg-5; tar -xf jtreg-5.1-b01.tar.gz -C jtreg-5 --strip-components 1; \ + rm jtreg*.tar.gz; + +# AdoptOpenJDK 8/10/11/16 +RUN mkdir -p /usr/lib/jvm && cd /usr/lib/jvm; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK8U-jdk_x64_linux_hotspot_8u292b10.tar.gz -O jdk-8.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK10U-jdk_x64_linux_hotspot_10.0.2_13.tar.gz -O jdk-10.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK11U-jdk_x64_linux_hotspot_11.0.11_9.tar.gz -O jdk-11.tar.gz; \ + wget -q http://121.36.53.23/bisheng_riscv/OpenJDK16U-jdk_x64_linux_hotspot_16.0.1_9.tar.gz -O jdk-16.tar.gz; \ + for tar in *.tar.gz; do tar xf $tar; done; \ + rm *.tar.gz; \ + ln -s jdk8* jdk-8; \ + ln -s jdk-10* jdk-10; \ + ln -s jdk-11* jdk-11; \ + ln -s jdk-16* jdk-16; + +ENV \ + JDK8_BOOT_DIR="/usr/lib/jvm/jdk-8" \ + JDK10_BOOT_DIR="/usr/lib/jvm/jdk-10" \ + JDK11_BOOT_DIR="/usr/lib/jvm/jdk-11" \ + JDK16_BOOT_DIR="/usr/lib/jvm/jdk-16" \ + JTREG4_DIR="/opt/jtreg-4" \ + JTREG5_DIR="/opt/jtreg-5" + +# Jenkins Agent +RUN set -eux; \ + yum install -y python3 python3-pip cpio bsdtar expect openssh sudo shadow curl wget vim git strace + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] diff --git a/src/dockerfile/jenkins-openeuler b/src/dockerfile/ci-common similarity index 74% rename from src/dockerfile/jenkins-openeuler rename to src/dockerfile/ci-common index 98ffa419b8a475cb358da0c1b71288e2162e4a9c..b99d662db71ce600fa0b4937cfb89491ecc30cc5 100644 --- a/src/dockerfile/jenkins-openeuler +++ b/src/dockerfile/ci-common @@ -1,5 +1,8 @@ # replace VERSION before run -FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler-VERSION/openjdk:11-jdk-stretch +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +RUN set -eux; \ + yum install -y python3 python3-pip cpio bsdtar expect openssh sudo shadow curl wget vim git strace ARG VERSION=4.3 ARG user=jenkins @@ -13,7 +16,9 @@ RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins- && chmod 644 /usr/share/jenkins/agent.jar \ && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar -COPY jenkins-agent /usr/local/bin/jenkins-agent +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + RUN chmod a+rx /usr/local/openjdk-11 \ && chmod a+rx /usr/local/bin/jenkins-agent \ && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave @@ -33,5 +38,3 @@ WORKDIR ${AGENT_WORKDIR} ENTRYPOINT ["jenkins-agent"] - -# swr.cn-north-4.myhuaweicloud.com/openeuler/jenkins/openeuler:base diff --git a/src/dockerfile/ci-soe-base b/src/dockerfile/ci-soe-base new file mode 100644 index 0000000000000000000000000000000000000000..b6332ad5480ac339a5a788f7568d377ab8ebd834 --- /dev/null +++ b/src/dockerfile/ci-soe-base @@ -0,0 +1,47 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +# golint not ready yet +RUN set -eux; \ + yum install -y curl sudo cpio bsdtar expect openssh shadow vim git strace python3 python3-pip build osc pylint + +# libabigail perl-XML-Structured not ready yet, use SP2 dailybuild +RUN set -eux; \ + arch="$(arch)"; \ + yum install -y http://121.36.53.23/SP2/libabigail-1.6-4.oe1.$arch.rpm; \ + yum install -y http://121.36.53.23/SP2/perl-XML-Structured-1.3-2.oe1.$arch.rpm + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] diff --git a/src/dockerfile/obs b/src/dockerfile/ci-ubuntu similarity index 100% rename from src/dockerfile/obs rename to src/dockerfile/ci-ubuntu diff --git a/src/dockerfile/openjdk-openeuler b/src/dockerfile/openjdk-openeuler index aaf591440970344bdbaf2421b89b9eb8caef92bc..7ca38a966437f405a91e46b429efeafd59e141a7 100644 --- a/src/dockerfile/openjdk-openeuler +++ b/src/dockerfile/openjdk-openeuler @@ -2,7 +2,7 @@ FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openeuler:VERSION RUN set -eux; \ - yum install -y tar wget python3 expect osc vim openssh shadow git + yum install -y tar wget # Default to UTF-8 file.encoding ENV LANG C.UTF-8 @@ -33,9 +33,11 @@ RUN set -eux; \ # this "case" statement is generated via "update.sh" case "$arch" in \ # arm64v8 - arm64 | aarch64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_aarch64_linux_11.0.8_10.tar.gz ;; \ + arm64 | aarch64) downloadUrl=http://121.36.53.23/AdoptOpenJDK/openjdk_aarch64.tgz ;; \ # amd64 - amd64 | i386:x86-64 | x86_64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_x64_linux_11.0.8_10.tar.gz ;; \ + amd64 | i386:x86-64 | x86_64) downloadUrl=http://121.36.53.23/AdoptOpenJDK/openjdk_x64.tgz ;; \ + #arm64 | aarch64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_aarch64_linux_11.0.8_10.tar.gz ;; \ + #amd64 | i386:x86-64 | x86_64) downloadUrl=https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.8%2B10/OpenJDK11U-jdk_x64_linux_11.0.8_10.tar.gz ;; \ # fallback *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ esac; \ diff --git a/src/dockerfile/release-tools-dockerfile b/src/dockerfile/release-tools-dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..e4976dce236cf2b6814834f8f336a8ebe0b2dcf0 --- /dev/null +++ b/src/dockerfile/release-tools-dockerfile @@ -0,0 +1,40 @@ +# replace VERSION before run +FROM swr.cn-north-4.myhuaweicloud.com/openeuler/openjdk/OPENJDK:TAG + +RUN set -eux; \ + yum install -y python-jenkins python3-requests python-concurrent-log-handler python3-gevent python3-marshmallow python3-pyyaml python-pandas python-xlrd python-retrying python-esdk-obs-python git + +ARG VERSION=4.3 +ARG user=jenkins +ARG group=jenkins +ARG uid=1000 +ARG gid=1000 +ARG AGENT_WORKDIR=/home/${user}/agent + +RUN curl --create-dirs -fsSLo /usr/share/jenkins/agent.jar https://repo.jenkins-ci.org/public/org/jenkins-ci/main/remoting/${VERSION}/remoting-${VERSION}.jar \ + && chmod 755 /usr/share/jenkins \ + && chmod 644 /usr/share/jenkins/agent.jar \ + && ln -sf /usr/share/jenkins/agent.jar /usr/share/jenkins/slave.jar + +RUN curl --create-dirs -fsSLo /usr/local/bin/jenkins-agent http://121.36.53.23/AdoptOpenJDK/jenkins-agent +#COPY jenkins-agent /usr/local/bin/jenkins-agent + +RUN chmod a+rx /usr/local/openjdk-11 \ + && chmod a+rx /usr/local/bin/jenkins-agent \ + && ln -s /usr/local/bin/jenkins-agent /usr/local/bin/jenkins-slave + +RUN groupadd -g ${gid} ${group} +RUN useradd -c "Jenkins user" -d /home/${user} -u ${uid} -g ${gid} -m ${user} +RUN echo "${user} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + + +USER ${user} +ENV AGENT_WORKDIR=${AGENT_WORKDIR} +RUN mkdir /home/${user}/.jenkins && mkdir -p ${AGENT_WORKDIR} + +VOLUME /home/${user}/.jenkins +VOLUME ${AGENT_WORKDIR} +WORKDIR ${AGENT_WORKDIR} + + +ENTRYPOINT ["jenkins-agent"] diff --git a/src/jobs/jenkins_job.py b/src/jobs/jenkins_job.py index 7ecab8a31bad19e651950d2bca4fab60e769149d..811058c3ca2d0f8c7db6ff0b3b6489283e86632c 100755 --- a/src/jobs/jenkins_job.py +++ b/src/jobs/jenkins_job.py @@ -59,7 +59,7 @@ class JenkinsJobs(object): :param interval: 每次batch请求后sleep时间（秒）， :return: """ - logger.info("{} jobs {}".format(action, jobs)) + logger.info("%s jobs %s", action, jobs) real_jobs = self.get_real_target_jobs(jobs, exclude_jobs if exclude_jobs else []) def run_once(target_jobs): @@ -68,17 +68,17 @@ class JenkinsJobs(object): """ batch = (len(target_jobs) + concurrency - 1) / concurrency _failed_jobs = [] - for index in xrange(batch): + for index in range(batch): works = [gevent.spawn(self.dispatch, action, job, jenkins_proxy) for job in target_jobs[index * concurrency: (index + 1) * concurrency]] - logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) + logger.info("%s works, %s/%s ", len(works), index + 1, batch) gevent.joinall(works) for work in works: if work.value["result"]: - logger.info("{} job {} ... ok".format(action, work.value["job"])) + logger.info("%s job %s ... ok", action, work.value["job"]) else: _failed_jobs.append(work.value["job"]) - logger.error("{} job {} ... failed".format(action, work.value["job"])) + logger.error("%s job %s ... failed", action, work.value["job"]) time.sleep(interval) @@ -86,15 +86,15 @@ class JenkinsJobs(object): failed_jobs = run_once(real_jobs) - for index in xrange(retry): + for index in range(retry): if not failed_jobs: break - logger.info("{} jobs failed, retrying {}/{}".format(len(failed_jobs), index + 1, retry)) + logger.info("%s jobs failed, retrying %s/%s", len(failed_jobs), index + 1, retry) failed_jobs = run_once(failed_jobs) if failed_jobs: - logger.warning("{} failed jobs".format(len(failed_jobs))) - logger.warning("{}{}".format(",".join(failed_jobs[:100]), "..." if len(failed_jobs) > 100 else "")) + logger.warning("%s failed jobs", len(failed_jobs)) + logger.warning("%s%s", ",".join(failed_jobs[:100]), "..." if len(failed_jobs) > 100 else "") def dispatch(self, action, job, jenkins_proxy): """ @@ -148,7 +148,7 @@ class SrcOpenEulerJenkinsJobs(JenkinsJobs): with open(os.path.join(exclusive_arch_path, filename), "r") as f: arches = f.readline() self._exclusive_arch[filename] = [arch.strip() for arch in arches.split(",")] - logger.debug("exclusive arch: {}".format(self._exclusive_arch)) + logger.debug("exclusive arch: %s", self._exclusive_arch) def get_real_target_jobs(self, jobs, exclude_jobs): """ @@ -274,7 +274,7 @@ class OpenEulerJenkinsJobs(SrcOpenEulerJenkinsJobs): # build script = self.guess_build_script(buddy["repo"]) - logger.debug("guess build script: {}".format("script")) + logger.debug("guess build script: %s", "script") ele = root.findall("buiders/hudson.task.Shell/command") if ele: # replace first command diff --git a/src/jobs/obs_meta_strategy.py b/src/jobs/obs_meta_strategy.py index 2b384265f371d76ea597f9d9065f886651351df0..24b0c40c91932c8a8426deedf16d94822ef36475 100755 --- a/src/jobs/obs_meta_strategy.py +++ b/src/jobs/obs_meta_strategy.py @@ -72,27 +72,27 @@ class ObsMetaStrategy(object): index = 0 for dirpath, dirnames, filenames in os.walk(self._obs_meta_path): # 忽略.osc目录 - if re.search("\.osc|\.git", dirpath): + if re.search("\.osc|\.git|\:Bak", dirpath): continue for filename in filenames: if filename == "_service": _service = os.path.join(dirpath, filename) try: - logger.debug("analysis {}".format(_service)) + logger.debug("analysis %s", _service) tree = ET.parse(_service) elements = tree.findall(".//param[@name=\"url\"]") # next/openEuler/zip except: - logger.exception("invalid xml format, {}".format(_service)) + logger.exception("invalid xml format, %s", _service) continue _repos = [element.text.strip("/").split("/")[-1] for element in elements] # eg: next/openEuler/zip - logger.debug("get repos: {}".format(_repos)) + logger.debug("get repos: %s", _repos) if any([repo in repos for repo in _repos]): package = dirpath.strip("/").split("/")[-1] # eg: master/openEuler:Mainline/zip/_services index += 1 - logger.info("{} {}...ok".format(index, _service)) - logger.info("package: {}, repos: {}".format(package, _repos)) + logger.info("%s %s...ok", index, _service) + logger.info("package: %s, repos: %s", package, _repos) for repo in _repos: self._package_repo[package].add(repo) self._repo_package[repo].add(package) diff --git a/src/jobs/repo_mapping.py b/src/jobs/repo_mapping.py index 49b1765bd79b1b5aafa4a27a299d540a43b91669..318f2b2ef1be7e189f9d437bde97b96d1df8501a 100755 --- a/src/jobs/repo_mapping.py +++ b/src/jobs/repo_mapping.py @@ -38,9 +38,9 @@ class RepoMapping(object): self._exclude_repos = kwargs.get("exclude_jobs") if kwargs.get("exclude_jobs") else [] self._repo_mapping = {} # 保存结果 self._ignored_repos = self._load_ignore_repo(ignored_repos_path, ignored_repos_key) - logger.debug("ignored repos: {}".format(self._ignored_repos)) + logger.debug("ignored repos: %s", self._ignored_repos) self._community_repos = self._load_community_repo(community_path) # 社区repos - logger.debug("community repos: {}".format(self._community_repos)) + logger.debug("community repos: %s", self._community_repos) @staticmethod def _load_ignore_repo(conf_file, ignored_repos_key): @@ -55,7 +55,7 @@ class RepoMapping(object): handler = yaml.safe_load(f) return handler.get(ignored_repos_key, []) except IOError as e: - logger.warning("{} not exist".format(conf_file)) + logger.warning("%s not exist", conf_file) return [] @staticmethod @@ -69,10 +69,10 @@ class RepoMapping(object): with open(community_path, "r") as f: handler = yaml.safe_load(f) repos = {item["name"]: item["type"] for item in handler["repositories"]} - logger.info("repos from community: {}".format(len(repos))) + logger.info("repos from community: %s", len(repos)) return repos except IOError as e: - logger.warning("{} not exist".format(community_path)) + logger.warning("%s not exist", community_path) return [] def _is_valid_repo(self, repo): diff --git a/src/proxy/es_proxy.py b/src/proxy/es_proxy.py index b439951c1fed5de95ebb7c8a6829253b8e321e9c..3b18141e95d30599fe95d6366a04d4cecfc68b4d 100644 --- a/src/proxy/es_proxy.py +++ b/src/proxy/es_proxy.py @@ -46,10 +46,10 @@ class ESProxy(object): :return: """ try: - logger.debug("es insert: {}".format(body)) + logger.debug("es insert: %s", body) rs = self._es.index(index, body=body) - logger.debug("insert result: {}".format(rs)) + logger.debug("insert result: %s", rs) return rs["result"] == "created" except elasticsearch.ElasticsearchException: logger.exception("elastic search insert document exception") @@ -62,9 +62,9 @@ class ESProxy(object): :param body: :return: """ - logger.debug("es search: {}".format(body)) + logger.debug("es search: %s", body) rs = self._es.search(index=index, body=body) - logger.debug("result: {}".format(rs)) + logger.debug("result: %s", rs) return rs['hits']['hits'] @@ -89,9 +89,9 @@ class ESProxy(object): """ try: body = {"query": query, "script": script} - logger.debug("es update: {}".format(body)) + logger.debug("es update: %s", body) rs = self._es.update_by_query(index, body=body) - logger.debug("update result: {}".format(rs)) + logger.debug("update result: %s", rs) return True except elasticsearch.ElasticsearchException: diff --git a/src/proxy/git_proxy.py b/src/proxy/git_proxy.py index e2c644bc8f40d13540fe798d9263e0c06e6f09e9..a7bca0b78f44be3a4fdff027e91f4a9c3fd7f897 100644 --- a/src/proxy/git_proxy.py +++ b/src/proxy/git_proxy.py @@ -1,7 +1,24 @@ # -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2020-09-23 +# Description: git api proxy +# ********************************************************************************** +""" + import os import logging -from cStringIO import StringIO +from io import StringIO import retrying from src.utils.shell_cmd import shell_cmd_live @@ -33,7 +50,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(init_cmd) if ret: - logger.warning("init repository failed, {}".format(ret)) + logger.warning("init repository failed, %s", ret) return None return cls(repo_dir) @@ -53,8 +70,8 @@ class GitProxy(object): self._repo_dir, "--progress" if progress else "", depth, url, pull_request, pull_request) ret, out, _ = shell_cmd_live(fetch_cmd, cap_out=True, cmd_verbose=False) if ret: - logger.error("git fetch failed, {}".format(ret)) - logger.error("{}".format(out)) + logger.error("git fetch failed, %s", ret) + logger.error("%s", out) return False return True @@ -69,7 +86,7 @@ class GitProxy(object): get_content_cmd = "cd {}; git show {}:{}".format(self._repo_dir, commit, file_path) ret, out, _ = shell_cmd_live(get_content_cmd, cap_out=True) if ret: - logger.warning("get file content of commit failed, {}".format(ret)) + logger.warning("get file content of commit failed, %s", ret) return None f = StringIO() @@ -89,7 +106,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(diff_files_cmd, cap_out=True) if ret: - logger.error("get diff files of commits failed, {}".format(ret)) + logger.error("get diff files of commits failed, %s", ret) return [] return out @@ -104,7 +121,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(extract_file_cmd, cap_out=True) if ret: - logger.error("extract diff files of patch failed, {}".format(ret)) + logger.error("extract diff files of patch failed, %s", ret) return [] return [line.split()[-1] for line in out] @@ -120,7 +137,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(apply_patch_cmd) if ret: - #logger.error("apply patch failed, {}".format(ret)) + #logger.error("apply patch failed, %s", ret) return False return True @@ -139,7 +156,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(apply_patch_cmd) if ret: - #logger.error("apply patch failed, {}".format(ret)) + #logger.error("apply patch failed, %s", ret) return False return True @@ -154,7 +171,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(get_commit_cmd, cap_out=True) if ret: - logger.error("get commit id of index failed, {}".format(ret)) + logger.error("get commit id of index failed, %s", ret) return None return out[0] @@ -169,7 +186,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(checkout_cmd) if ret: - logger.warning("checkout failed, {}".format(ret)) + logger.warning("checkout failed, %s", ret) return False return True @@ -184,7 +201,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(checkout_cmd) if ret: - logger.warning("checkout failed, {}".format(ret)) + logger.warning("checkout failed, %s", ret) return False return True @@ -207,7 +224,7 @@ class GitProxy(object): ret, out, _ = shell_cmd_live(tree_hashes_cmd, cap_out=True) if ret: - logger.error("get tree hashes failed, {}".format(ret)) + logger.error("get tree hashes failed, %s", ret) return None return out @@ -226,7 +243,7 @@ class GitProxy(object): ret, _, _ = shell_cmd_live(fetch_cmd) if ret: - logger.error("fetch failed, {}".format(ret)) + logger.error("fetch failed, %s", ret) return False return True diff --git a/src/proxy/gitee_proxy.py b/src/proxy/gitee_proxy.py index a46659a0407673a27f3333bf4ddbe5ed03d7f371..3cfa96f952ee5aaa37150ff05906a490b3e56cc5 100644 --- a/src/proxy/gitee_proxy.py +++ b/src/proxy/gitee_proxy.py @@ -35,7 +35,7 @@ class GiteeProxy(object): :param comment: 评论内容 :return: 0成功，其它失败 """ - logger.debug("comment pull request {}".format(pr)) + logger.debug("comment pull request %s", pr) comment_pr_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/comments".format(self._owner, self._repo, pr) data = {"access_token": self._token, "body": comment} @@ -58,7 +58,7 @@ class GiteeProxy(object): logger.debug("create tags, but no tags") return True - logger.debug("create tags {} of pull request {}".format(tags, pr)) + logger.debug("create tags %s of pull request %s", tags, pr) pr_tag_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/labels?access_token={}".format( self._owner, self._repo, pr, self._token) @@ -81,7 +81,7 @@ class GiteeProxy(object): logger.debug("replace tags, but no tags") return True - logger.debug("replace all tags with {} of pull request {}".format(tags, pr)) + logger.debug("replace all tags with %s of pull request %s", tags, pr) pr_tag_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/labels?access_token={}".format( self._owner, self._repo, pr, self._token) @@ -99,7 +99,7 @@ class GiteeProxy(object): :param tag: 标签 :return: 0成功，其它失败 """ - logger.debug("delete tag {} of pull request {}".format(tag, pr)) + logger.debug("delete tag %s of pull request %s", tag, pr) pr_tag_url = "https://gitee.com/api/v5/repos/{}/{}/pulls/{}/labels/{}?access_token={}".format( self._owner, self._repo, pr, tag, self._token) @@ -128,7 +128,7 @@ class GiteeProxy(object): """ handler = yaml.safe_load(response.text) repos.update({item["name"]: item["type"] for item in handler["repositories"]}) - logger.info("repos from community: {}".format(len(repos))) + logger.info("repos from community: %s", len(repos)) community_repo_url = "https://gitee.com/openeuler/community/raw/master/repository/src-openeuler.yaml" logger.info("requests repos from community, this will take multi seconds") @@ -143,7 +143,7 @@ class GiteeProxy(object): :param state: pr状态 :return: str or None """ - logger.debug("get last pull request committer, branch: {}, state: {}".format(branch, state)) + logger.debug("get last pull request committer, branch: %s, state: %s", branch, state) pr_url = "https://gitee.com/api/v5/repos/{}/{}/pulls?access_token={}&state={}&base={}" \ "&page=1&per_page=1".format(self._owner, self._repo, self._token, state, branch) @@ -161,7 +161,7 @@ class GiteeProxy(object): if handler: try: committer[0] = handler[0]["user"]["login"] - logger.debug("get last pr committer: {}".format(committer)) + logger.debug("get last pr committer: %s", committer) except KeyError: logger.exception("extract committer info from gitee exception") @@ -170,4 +170,4 @@ class GiteeProxy(object): if rs != 0: logger.warning("get last pr committer failed") - return committer[0] + return committer[0] \ No newline at end of file diff --git a/src/proxy/jenkins_patch.py b/src/proxy/jenkins_patch.py index 4fa3ef2b527651a66eb15be7476e3bed37cd75f9..1b73d44039ba7aabb985e946ade50b5778943758 100644 --- a/src/proxy/jenkins_patch.py +++ b/src/proxy/jenkins_patch.py @@ -1,5 +1,5 @@ # -*- encoding=utf-8 -*- -from urllib import quote as urlquote +from urllib.parse import quote from jenkinsapi.jenkinsbase import JenkinsBase @@ -11,7 +11,7 @@ def resolve_job_folders(self, jobs): jobs.remove(job) jobs += self.process_job_folder(job, self.baseurl) else: - job["url"] = '%s/job/%s' % (self.baseurl, urlquote(job['name'])) + job["url"] = '%s/job/%s' % (self.baseurl, quote(job['name'])) return jobs diff --git a/src/proxy/jenkins_proxy.py b/src/proxy/jenkins_proxy.py index b4b035bd1cc2a9c1c9629217dacc468a1ea7543b..ea40dfcd68a9da382b089d1811687c72e35f858f 100644 --- a/src/proxy/jenkins_proxy.py +++ b/src/proxy/jenkins_proxy.py @@ -55,7 +55,7 @@ class JenkinsProxy(object): self._jenkins.create_job(job, config) return True except Exception as e: - logger.exception("create job exception, {}".format(e)) + logger.exception("create job exception, %s", e) return False def update_job(self, job, config): @@ -70,7 +70,7 @@ class JenkinsProxy(object): jks_job.update_config(config) return True except Exception as e: - logger.exception("update job exception, {}".format(e)) + logger.exception("update job exception, %s", e) return False def get_config(self, job): @@ -82,7 +82,7 @@ class JenkinsProxy(object): try: return self._jenkins[job].get_config() except Exception as e: - logger.exception("get config exception, {}".format(e)) + logger.exception("get config exception, %s", e) return None def get_build(self, job, build_no): @@ -95,7 +95,7 @@ class JenkinsProxy(object): try: return self._jenkins[job].get_build(build_no) except Exception as e: - logger.exception("get job build exception, {}".format(e)) + logger.exception("get job build exception, %s", e) return None @classmethod @@ -109,7 +109,7 @@ class JenkinsProxy(object): parent_build = build.get_upstream_build() return parent_build.get_upstream_build() if parent_build else None except Exception as e: - logger.exception("get grandpa build exception, {}".format(e)) + logger.exception("get grandpa build exception, %s", e) return None def _get_upstream_jobs(self, job): @@ -119,22 +119,22 @@ class JenkinsProxy(object): :param job: Jenkins Job object :return: """ - logger.debug("get upstream jobs of {}".format(job._data["fullName"])) + logger.debug("get upstream jobs of %s", job._data["fullName"]) jobs = [] for project in job._data["upstreamProjects"]: # but is the only way of get upstream projects info url = project.get("url") name = project.get("name") - logger.debug("upstream project: {} {}".format(url, name)) + logger.debug("upstream project: %s %s", url, name) m = re.match("(.*)/job/.*", url) # remove last part of job url, greedy match base_url = m.group(1) - logger.debug("base url {}".format(base_url)) + logger.debug("base url %s", base_url) try: j = jenkins.Jenkins(base_url, self._username, self._token, timeout=self._timeout) jobs.append(j[name]) except Exception as e: - logger.exception("get job of {} exception".format(url)) + logger.exception("get job of %s exception", url) continue return jobs @@ -167,17 +167,17 @@ class JenkinsProxy(object): cause_build = cause_job.get_build(cause_build_id) cause_cause_build_id = cause_build.get_upstream_build_number() - logger.debug("cause_build_id: {}, cause_job_name: {}, cause_cause_build_id: {}".format( - cause_build_id, cause_job_name, cause_cause_build_id)) + logger.debug("cause_build_id: %s, cause_job_name: %s, cause_cause_build_id: %s", + cause_build_id, cause_job_name, cause_cause_build_id) upstream_builds = [] for upstream_job in upstream_jobs: - logger.debug("{}".format(upstream_job._data["fullName"])) + logger.debug("%s", upstream_job._data["fullName"]) for build_id in upstream_job.get_build_ids(): - logger.debug("try build id {}".format(build_id)) + logger.debug("try build id %s", build_id) a_build = upstream_job.get_build(build_id) if a_build.get_upstream_build_number() == cause_cause_build_id: - logger.debug("build id {} match".format(build_id)) + logger.debug("build id %s match", build_id) upstream_builds.append(a_build) break diff --git a/src/proxy/kafka_proxy.py b/src/proxy/kafka_proxy.py index 9ff7ff74e87da603a255570bf399a64f00c9fbeb..110b46e8b0a70593954c4b9e60881472b1e26e4b 100644 --- a/src/proxy/kafka_proxy.py +++ b/src/proxy/kafka_proxy.py @@ -37,6 +37,7 @@ class KafkaProducerProxy(object): self._timeout = timeout self._kp = kafka.KafkaProducer(bootstrap_servers=brokers, + key_serializer=str.encode, value_serializer=lambda v:json.dumps(v).encode("utf-8")) def send(self, topic, key=None, value=None): @@ -48,12 +49,12 @@ class KafkaProducerProxy(object): :return: """ try: - logger.debug("kafka send: {}, {}".format(key, value)) + logger.debug("kafka send: %s, %s", key, value) future = self._kp.send(topic, value=value, key=key) rs = future.get(timeout=self._timeout) - logger.debug("kafka send result: {}".format(rs)) + logger.debug("kafka send result: %s", rs) return True except errors.KafkaTimeoutError: logger.exception("kafka send timeout exception") diff --git a/src/proxy/obs_proxy.py b/src/proxy/obs_proxy.py index eee8ad139bcdf070d7565678d533e871052c8cc1..98fd978172a057cd4a855ce5d14150e611fa8873 100644 --- a/src/proxy/obs_proxy.py +++ b/src/proxy/obs_proxy.py @@ -45,29 +45,35 @@ class OBSProxy(object): cmd = "osc ll {} {}".format(project, package) ret, rs, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.error("list project package error, {}".format(ret)) + logger.error("list project package error, %s", ret) return None return rs @staticmethod - def list_repos_of_arch(project, package, arch): + def list_repos_of_arch(project, package, arch, show_exclude=False): """ 获取包的repo列表 :param project: :param package: :return: """ - cmd = "osc results {} {} -a {}".format(project, package, arch) + cmd = "osc results {} {} {} -a {}".format( + "--show-exclude" if show_exclude else "", project, package, arch) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.debug("list obs repos of arch error, {}".format(ret)) + logger.debug("list obs repos of arch error, %s", ret) return [] rs = [] for line in out: - repo, arch, state = line.split() - rs.append({"repo": repo, "state": state}) + try: + repo, arch, state = line.split() + mpac = package + except ValueError: + repo, arch, pac, state = line.split() + mpac = pac.split(":")[-1] + rs.append({"repo": repo, "mpac": mpac, "state": state}) return rs @@ -82,7 +88,7 @@ class OBSProxy(object): cmd = "osc results {} --csv |grep {} | awk -F';' '{{print $1}}'".format(project, state) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.debug("list package of state error, {}".format(ret)) + logger.debug("list package of state error, %s", ret) return [] return out @@ -99,36 +105,41 @@ class OBSProxy(object): _ = os.path.isdir(project) and shutil.rmtree(project) cmd = "osc co {} {}".format(project, package) - logger.info("osc co {} {}".format(project, package)) + logger.info("osc co %s %s", project, package) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: - logger.error("checkout package error, {}".format(ret)) + logger.error("checkout package error, %s", ret) return False return True @staticmethod - def build_package(project, package, repo, arch, debug=False): + def build_package(project, package, repo, arch, spec, mpac, debug=False, root_build=False, disable_cpio=False): """ build :param project: :param package: :param repo: :param arch: + :param spec: + :param mpac: multibuild package :param debug: :return: """ package_path = "{}/{}".format(project, package) - cmd = "cd {}; osc build {} {} {} --no-verify --clean".format( - package_path, repo, arch, "--disable-debuginfo" if not debug else "") - - logger.info("osc build {} {} {} --no-verify --clean".format( - repo, arch, "--disable-debuginfo" if not debug else "")) + root_opt = "--userootforbuild" if root_build else "" + debuginfo_opt = "--disable-debuginfo" if not debug else "" + disable_cpio_bulk = "--disable-cpio-bulk-download" if disable_cpio else "" + cmd = "cd {}; osc build {} {} {} {} {} {} --no-verify --clean --noservice -M {}".format( + package_path, repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) + + logger.info("osc build %s %s %s %s %s %s --no-verify --clean --noservice -M %s", + repo, arch, spec, root_opt, debuginfo_opt, disable_cpio_bulk, mpac) ret, _, _ = shell_cmd_live(cmd, verbose=True) if ret: - logger.error("build package error, {}".format(ret)) + logger.error("build package error, %s", ret) return False return True @@ -146,7 +157,7 @@ class OBSProxy(object): cmd = "osc api /build/{}/{}/{}/{}/_history".format(project, repo, arch, package) ret, out, _ = shell_cmd_live(cmd, cap_out=True) if ret: - logger.debug("list build history of package error, {}".format(ret)) + logger.debug("list build history of package error, %s", ret) return "" return "\n".join(out) diff --git a/src/proxy/requests_proxy.py b/src/proxy/requests_proxy.py index b88129fbc83a20f050f53de528f9a0f32955ca4f..87de8296a93b575e9a666a00c301b0d1f8d8ce4a 100644 --- a/src/proxy/requests_proxy.py +++ b/src/proxy/requests_proxy.py @@ -38,9 +38,9 @@ def do_requests(method, url, querystring=None, body=None, auth=None, timeout=30, :return: """ try: - logger.debug("http requests, {} {} {}".format(method, url, timeout)) - logger.debug("querystring: {}".format(querystring)) - logger.debug("body: {}".format(body)) + logger.debug("http requests, %s %s %s", method, url, timeout) + logger.debug("querystring: %s", querystring) + logger.debug("body: %s", body) if method.lower() not in ["get", "post", "put", "delete"]: return -1 @@ -60,7 +60,7 @@ def do_requests(method, url, querystring=None, body=None, auth=None, timeout=30, else: rs = func(url, timeout=timeout) - logger.debug("status_code {}".format(rs.status_code)) + logger.debug("status_code %s", rs.status_code) if rs.status_code not in [requests.codes.ok, requests.codes.created, requests.codes.no_content]: return 1 @@ -78,11 +78,11 @@ def do_requests(method, url, querystring=None, body=None, auth=None, timeout=30, return 0 except requests.exceptions.SSLError as e: - logger.warning("requests {} ssl exception, {}".format(url, e)) + logger.warning("requests %s ssl exception, %s", url, e) return -2 except requests.exceptions.Timeout as e: logger.warning("requests timeout") return 2 except requests.exceptions.RequestException as e: - logger.warning("requests exception, {}".format(e)) + logger.warning("requests exception, %s", e) return 3 diff --git a/src/requirements b/src/requirements index bb5b47e47ef3df8a08b269e06c174222d09b72cf..7d34070afd36a0242b173d47d037f55727678ba0 100644 --- a/src/requirements +++ b/src/requirements @@ -3,11 +3,12 @@ jenkinsapi colorlog threadpool PyYAML -gevent==1.2.2 +gevent jsonpath mock tldextract chardet -kafka +kafka-python elasticsearch retrying +scanoss diff --git a/src/tools/obs_package_build_history.py b/src/tools/obs_package_build_history.py index cd0863785e49dae50e791b254884c10227969e8c..ba9a8d08df1e577d2304594733ab8f98178f574d 100644 --- a/src/tools/obs_package_build_history.py +++ b/src/tools/obs_package_build_history.py @@ -46,7 +46,7 @@ class JobBuildHistory(object): try: root = ET.fromstring(history) except ParseError: - logger.exception("package: {}, build history: {}".format(package, history)) + logger.exception("package: %s, build history: %s", package, history) return {"package": package, "max": 0, "min": 0, "average": 0, "times": -1} duration = [int(ele.get("duration")) for ele in root.findall("entry")] @@ -74,11 +74,11 @@ class JobBuildHistory(object): for index in range(batch): works = [gevent.spawn(JobBuildHistory.get_package_job_duration, project, package, repo, arch) for package in packages[index * concurrency: (index + 1) * concurrency]] - logger.info("{} works, {}/{} ".format(len(works), index + 1, batch)) + logger.info("%s works, %s/%s ", len(works), index + 1, batch) gevent.joinall(works) for work in works: - logger.debug("{}: {}".format(work.value["package"], work.value)) - logger.info("{} ...done".format(work.value["package"])) + logger.debug("%s: %s", work.value["package"], work.value) + logger.info("{%s} ...done", work.value["package"]) rs.append(work.value) time.sleep(1) diff --git a/src/tools/obs_package_build_report.py b/src/tools/obs_package_build_report.py index 6c018bda35139d576c2d927397b809f55c6b951c..78e25e4aacd3a9a2c345379ed856ce16aaa8620e 100644 --- a/src/tools/obs_package_build_report.py +++ b/src/tools/obs_package_build_report.py @@ -45,12 +45,18 @@ class ObsPackageBuildReport(object): "openEuler:21.03": "openEuler-21.03", "openEuler:21.03:Epol": "openEuler-21.03", "openEuler:21.03:Extras": "openEuler-21.03", + "openEuler:21.09": "openEuler-21.09", + "openEuler:21.09:Epol": "openEuler-21.09", + "openEuler:21.09:Extras": "openEuler-21.09", "openEuler:20.03:LTS:SP2:oepkg:openstack:common": "oepkg_openstack-common_oe-20.03-LTS-SP2", "openEuler:20.03:LTS:SP2:oepkg:openstack:queens": "oepkg_openstack-queens_oe-20.03-LTS-SP2", "openEuler:20.03:LTS:SP2:oepkg:openstack:rocky": "oepkg_openstack-rocky_oe-20.03-LTS-SP2", "openEuler:20.03:LTS:Next:oepkg:openstack:common": "oepkg_openstack-common_oe-20.03-LTS-Next", "openEuler:20.03:LTS:Next:oepkg:openstack:queens": "oepkg_openstack-queens_oe-20.03-LTS-Next", - "openEuler:20.03:LTS:Next:oepkg:openstack:rocky": "oepkg_openstack-rocky_oe-20.03-LTS-Next" + "openEuler:20.03:LTS:Next:oepkg:openstack:rocky": "oepkg_openstack-rocky_oe-20.03-LTS-Next", + "openEuler:20.03:LTS:SP3:oepkg:openstack:common": "oepkg_openstack-common_oe-20.03-LTS-SP3", + "openEuler:20.03:LTS:SP3:oepkg:openstack:queens": "oepkg_openstack-queens_oe-20.03-LTS-SP3", + "openEuler:20.03:LTS:SP3:oepkg:openstack:rocky": "oepkg_openstack-rocky_oe-20.03-LTS-SP3" } GITEE_OWNER = "src-openeuler" @@ -94,17 +100,17 @@ class ObsPackageBuildReport(object): # try: # branch = self.__class__.PROJECT_BRANCH_MAPPING[self._project] # except KeyError: -# logger.exception("project {} not support".format(self._project)) +# logger.exception("project %s not support", self._project) # return branch = "master" # get packages in project of state packages = OBSProxy.list_packages_of_state(self._project, self._state) - logger.info("project {} state {}, find {} packages".format(self._project, self._state, len(packages))) + logger.info("project %s state %s, find %s packages", self._project, self._state, len(packages)) # get last pr committer for index, package in enumerate(packages): - logger.info("{}: {}".format(index, package)) + logger.info("%s: %s", index, package) gp = GiteeProxy(self.GITEE_OWNER, package, gitee_api_token) committer = gp.get_last_pr_committer(branch) real_name = self._real_name_mapping.get(committer, "N/A") diff --git a/src/utils/shell_cmd.py b/src/utils/shell_cmd.py index d3bc10cc92b5b8cb2678e8bd20326d23724438cd..e3be542913ca688279031c7a92656c579821eef8 100755 --- a/src/utils/shell_cmd.py +++ b/src/utils/shell_cmd.py @@ -8,12 +8,18 @@ no_fmt_logger = logging.getLogger("no_fmt") def shell_cmd(cmd, inmsg=None): - logger.debug("exec cmd -- [{}]".format(cmd)) + """ + 创建子进程执行命令，返回执行结果 + :param cmd: 命令 + :param inmsg: 输入 + :return: + """ + logger.debug("exec cmd -- [%s]", cmd) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if inmsg: p.stdin.write(inmsg) out, err = p.communicate() - logger.debug("iret: {}, rs: {}, err: {}".format(p.returncode, out, err)) + logger.debug("iret: %s, rs: %s, err: %s", p.returncode, out, err) return p.returncode, out, err @@ -29,7 +35,7 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False :return: """ if cmd_verbose: - logger.debug("exec cmd -- {}".format(cmd)) + logger.debug("exec cmd -- %s", cmd) p = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) if cap_in: @@ -42,6 +48,7 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False while True: line = p.stdout.readline() if line: + line = line.decode("utf-8", errors="ignore") line = line.strip() no_fmt_logger.info(line) if verbose else no_fmt_logger.debug(line) if cap_out and line and line != "\n": @@ -50,14 +57,18 @@ def shell_cmd_live(cmd, cap_in=None, cap_out=False, cap_err=False, verbose=False break if cap_out: - logger.debug("total {} lines output".format(len(out))) + logger.debug("total %s lines output", len(out)) ret = p.poll() err = None if ret: - logger.debug("return code {}".format(ret)) - err = p.stderr.read() - no_fmt_logger.error(err) if verbose else no_fmt_logger.debug(line) + logger.debug("return code %s", ret) + while True: + line= p.stderr.readline() + if not line: + break + err = line.decode("utf-8", errors="ignore").strip() + no_fmt_logger.error(err) if verbose else no_fmt_logger.debug(err) return ret, out, err if cap_err else None diff --git a/test/ac/acl/license/test_check_license.py b/test/ac/acl/license/test_check_license.py index 3ca7b26fe90513bf1096f7bb3be4277364ba8370..09c3f9d70c715c07de9dd091957d663de107a809 100644 --- a/test/ac/acl/license/test_check_license.py +++ b/test/ac/acl/license/test_check_license.py @@ -17,13 +17,14 @@ """ import unittest -import mock +from unittest import mock import sys import os import types import logging.config import logging import shutil +from src.ac.acl.package_license.package_license import PkgLicense from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS from src.ac.acl.package_license.check_license import CheckLicense @@ -52,9 +53,9 @@ class TestCheckPkgLicense(unittest.TestCase): self._pkg_license.load_config() def decompress(self): self._gr.decompress_all() - check.get_work_tar_dir = types.MethodType(get_work_tar_dir, check, CheckLicense) - check.load_license_config = types.MethodType(load_license_config, check, CheckLicense) - check.decompress = types.MethodType(decompress, check, CheckLicense) + check.get_work_tar_dir = types.MethodType(get_work_tar_dir, check) + check.load_license_config = types.MethodType(load_license_config, check) + check.decompress = types.MethodType(decompress, check) def _test_check_license_in_spec(self, dir_key, predict): os.chdir(os.path.join(self.DIR_PATH, @@ -64,9 +65,9 @@ class TestCheckPkgLicense(unittest.TestCase): self.bind_func(cl) cl.load_license_config() self.assertEqual(cl.check_license_in_spec(), predict) - + def test_check_license_in_spec_none(self): - self._test_check_license_in_spec("no_spec", WARNING) + self._test_check_license_in_spec("no_spec", FAILED) def test_check_license_in_spec_succeed(self): self._test_check_license_in_spec("spec_success", SUCCESS) @@ -89,8 +90,9 @@ class TestCheckPkgLicense(unittest.TestCase): shutil.rmtree(cl.get_work_tar_dir()) def test_check_license_none(self): - self._test_check_license_in_src("no_src", WARNING) - + # 源码中不存在license由日志保存，不返回失败结果 + self._test_check_license_in_src("no_src", SUCCESS) + def test_check_license_in_src_succeed(self): self._test_check_license_in_src("src_success", SUCCESS) @@ -119,6 +121,11 @@ class TestCheckPkgLicense(unittest.TestCase): def test_check_license_same_failed(self): self._test_check_license_same("spec_src_diff", WARNING) + def test_check_license_same_later_version(self): + cl = PkgLicense() + cl.load_config() + self.assertEqual(cl.check_licenses_is_same(["GPL-1.0-or-later"], ["GPL-3.0-only"], cl._later_support_license), True) + if __name__ == "__main__": work_dir = os.getcwd() diff --git a/test/ac/acl/openlibing/__init__.py b/test/ac/acl/openlibing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9d89286419afdb586290dd2cdf72aba400e6940c --- /dev/null +++ b/test/ac/acl/openlibing/__init__.py @@ -0,0 +1,17 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-08-03 +# Description: check spec file +# ********************************************************************************** +""" \ No newline at end of file diff --git a/test/ac/acl/openlibing/test_check_code.py b/test/ac/acl/openlibing/test_check_code.py new file mode 100644 index 0000000000000000000000000000000000000000..8f0623b6897c61adf356e32b6c4d325a61dbbbad --- /dev/null +++ b/test/ac/acl/openlibing/test_check_code.py @@ -0,0 +1,90 @@ +# -*- encoding=utf-8 -*- +""" +# ********************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: +# Create: 2021-08-03 +# Description: check spec file +# ********************************************************************************** +""" + + +import unittest +import os +import logging.config +import logging +import shutil + +from unittest import mock +from src.ac.framework.ac_result import FAILED, WARNING, SUCCESS +from src.ac.acl.openlibing.check_code import CheckCode + +logging.getLogger('test_logger') + + +class TestCheckCode(unittest.TestCase): + + CODECHECK = { + "codecheck_api_url": "http://124.71.75.234:8384/api/openlibing/codecheck/start", + "pr_url": "https://gitee.com/openeuler/pkgship/pulls/210", + "pr_number": "210", + "repo": "pkgship" + } + + def _test_check_code(self, predict): + cc = CheckCode('./test', self.CODECHECK.get("repo")) + self.assertEqual(cc(codecheck=self.CODECHECK), mock.Mock(return_value=predict)()) + + def _test_codecheck_api(self, return_value): + mock_fun = mock.Mock(return_value=return_value) + self.assertEqual(CheckCode.get_codecheck_result + (self.CODECHECK.get('pr_url'), self.CODECHECK.get("codecheck_api_url")), mock_fun()) + + def test_check_code_success(self): + self._test_check_code(SUCCESS) + + def test_codecheck_api_success(self): + return_value = (0, { + "code": "200", + "msg": "success", + "data": "http://124.71.75.234/inc/315fda2799c94eaf90fda86dfe1148c1/reports/" + + "90b92657bdc35b53481711ff6eac5f71/summary" + }) + self._test_codecheck_api(return_value) + + def test_codecheck_api_success_no_pr_url(self): + return_value = (0, { + "code": "400", + "msg": "pr_url is not null", + }) + self.CODECHECK['pr_url'] = "" + self._test_codecheck_api(return_value) + + def test_codecheck_api_success_no_fail(self): + return_value = (0, { + "code": "500", + "msg": "codecheck is error", + }) + self._test_codecheck_api(return_value) + + +if __name__ == "__main__": + work_dir = os.getcwd() + _ = not os.path.exists("log") and os.mkdir("log") + logger_conf_path = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), + "../../../../src/conf/logger.conf")) + logging.config.fileConfig(logger_conf_path) + logger = logging.getLogger("test_logger") + # Test Package License + suite = unittest.makeSuite(TestCheckCode) + unittest.TextTestRunner().run(suite) + os.chdir(work_dir) + shutil.rmtree("log") diff --git a/test/ac/acl/package_yaml/test_check_repo.py b/test/ac/acl/package_yaml/test_check_repo.py index 3de78eba7ff66786d466881b0fa3b14026f17a65..f1d75224d349a653cc0abd996a09a6899ead29a1 100644 --- a/test/ac/acl/package_yaml/test_check_repo.py +++ b/test/ac/acl/package_yaml/test_check_repo.py @@ -17,17 +17,20 @@ """ import unittest -import mock +from unittest import mock import os import yaml import logging.config import logging import shutil +from time import sleep from src.ac.acl.package_yaml.check_repo import ReleaseTagsFactory logging.getLogger('test_logger') +ACCESS2INTERNET = False + class TestGetReleaseTags(unittest.TestCase): TEST_YAML_DIR = { "hg": os.path.join(os.path.dirname(os.path.realpath(__file__)), "repo_test_sample/hg_test/hg_test.yaml"), @@ -48,58 +51,69 @@ class TestGetReleaseTags(unittest.TestCase): with open(filepath, 'r') as yaml_data: # load yaml data result = yaml.safe_load(yaml_data) except IOError as e: - logging.warning("package yaml not exist. {}".format(str(e))) + logging.warning("package yaml not exist. %s", str(e)) except yaml.YAMLError as exc: - logging.warning("Error parsering YAML: {}".format(str(exc))) + logging.warning("Error parsering YAML: %s", str(exc)) finally: return result def _get_test_tags(self, version): + sleep(2) yaml_content = self._load_yaml(self.TEST_YAML_DIR[version]) vc = yaml_content.get("version_control", "") sr = yaml_content.get("src_repo", "") release_tags = ReleaseTagsFactory.get_release_tags(vc) return release_tags.get_tags(sr) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_hg_release_tags(self): release_tags = self._get_test_tags("hg") self.assertEqual(len(release_tags) > 0, True) - def test_get_github_release_tags(self): - release_tags = self._get_test_tags("github") - self.assertEqual(len(release_tags) > 0, True) + # 当前测试用例中网址无法访问，待后续更新，暂时关闭该单测 + # def test_get_github_release_tags(self): + # release_tags = self._get_test_tags("github") + # self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_git_release_tags(self): release_tags = self._get_test_tags("git") self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_gitlab_gnome_release_tags(self): release_tags = self._get_test_tags("gitlab.gnome") self.assertEqual(len(release_tags) > 0, True) - + + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_svn_release_tags(self): release_tags = self._get_test_tags("svn") self.assertEqual(len(release_tags) > 0, True) - def test_get_metacpan_release_tags(self): - release_tags = self._get_test_tags("metacpan") - self.assertEqual(len(release_tags) > 0, True) + # 当前测试用例中网址无法访问，待后续更新，暂时关闭该单测 + # def test_get_metacpan_release_tags(self): + # release_tags = self._get_test_tags("metacpan") + # self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_pypi_release_tags(self): release_tags = self._get_test_tags("pypi") self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_rubygem_release_tags(self): release_tags = self._get_test_tags("rubygem") self.assertEqual(len(release_tags) > 0, True) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_get_gitee_release_tags(self): release_tags = self._get_test_tags("gitee") self.assertEqual(len(release_tags) > 0, True) - def test_get_gnu_ftp_release_tags(self): - release_tags = self._get_test_tags("gnu-ftp") - self.assertEqual(len(release_tags) > 0, True) + # 当前测试用例中网址无法访问，待后续更新，暂时关闭该单测 + # def test_get_gnu_ftp_release_tags(self): + # release_tags = self._get_test_tags("gnu-ftp") + # self.assertEqual(len(release_tags) > 0, True) if __name__ == '__main__': work_dir = os.getcwd() diff --git a/test/ac/acl/package_yaml/test_check_yaml.py b/test/ac/acl/package_yaml/test_check_yaml.py index 0a189bae62142e631354e0a3cb9100b3a95658e9..42ee752ea839c6d58fa65699af689e787d61fb0e 100644 --- a/test/ac/acl/package_yaml/test_check_yaml.py +++ b/test/ac/acl/package_yaml/test_check_yaml.py @@ -17,7 +17,7 @@ """ import unittest -import mock +from unittest import mock import sys import os import types @@ -37,6 +37,8 @@ from src.ac.acl.package_yaml.check_repo import HgReleaseTags, GithubReleaseTags, logging.getLogger('test_logger') +ACCESS2INTERNET = False + class TestCheckYamlField(unittest.TestCase): TEST_YAML_DIR = { @@ -48,7 +50,7 @@ class TestCheckYamlField(unittest.TestCase): self.cy = CheckPackageYaml("", "", "") def set_yaml(self, file): self._gr.yaml_file = file - self.cy.set_yaml = types.MethodType(set_yaml, self.cy, CheckPackageYaml) # python3该接口有变化 为实例动态绑定接口 + self.cy.set_yaml = types.MethodType(set_yaml, self.cy) def test_none_file(self): self.cy.set_yaml(None) @@ -104,21 +106,24 @@ class TestCheckYamlRepo(unittest.TestCase): self._gr.yaml_file = file def set_spec(self, file): self._spec = RPMSpecAdapter(file) - self.cy.set_yaml = types.MethodType(set_yaml, self.cy, CheckPackageYaml) # python3该接口有变化 为实例动态绑定接口 - self.cy.set_spec = types.MethodType(set_spec, self.cy, CheckPackageYaml) # python3该接口有变化 为实例动态绑定接口 + self.cy.set_yaml = types.MethodType(set_yaml, self.cy) + self.cy.set_spec = types.MethodType(set_spec, self.cy) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_none_file(self): self.cy.set_yaml(None) self.cy.check_fields() result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") def test_NA_repo(self): self.cy.set_yaml(self.TEST_YAML_DIR["na"]) self.cy.check_fields() result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(HgReleaseTags, "get_tags") def test_hg_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["hg"]) @@ -127,6 +132,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(HgReleaseTags, "get_tags") def test_hg_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["hg"]) @@ -135,6 +141,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GithubReleaseTags, "get_tags") def test_github_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["github"]) @@ -143,6 +150,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GithubReleaseTags, "get_tags") def test_github_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["github"]) @@ -151,6 +159,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GitReleaseTags, "get_tags") def test_git_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["git"]) @@ -159,6 +168,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GitReleaseTags, "get_tags") def test_git_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["git"]) @@ -167,6 +177,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GitlabReleaseTags, "get_tags") def test_gitlab_gnome_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gitlab.gnome"]) @@ -175,6 +186,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GitlabReleaseTags, "get_tags") def test_gitlab_gnome_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gitlab.gnome"]) @@ -183,6 +195,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(SvnReleaseTags, "get_tags") def test_svn_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["svn"]) @@ -191,6 +204,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(SvnReleaseTags, "get_tags") def test_svn_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["svn"]) @@ -199,6 +213,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(MetacpanReleaseTags, "get_tags") def test_metacpan_repoo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["metacpan"]) @@ -207,6 +222,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(MetacpanReleaseTags, "get_tags") def test_metacpan_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["metacpan"]) @@ -215,6 +231,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(PypiReleaseTags, "get_tags") def test_pypi_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["pypi"]) @@ -223,6 +240,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(PypiReleaseTags, "get_tags") def test_pypi_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["pypi"]) @@ -231,6 +249,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(RubygemReleaseTags, "get_tags") def test_rubygem_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["rubygem"]) @@ -239,6 +258,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(RubygemReleaseTags, "get_tags") def test_rubygem_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["rubygem"]) @@ -247,6 +267,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GiteeReleaseTags, "get_tags") def test_gitee_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gitee"]) @@ -255,6 +276,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GiteeReleaseTags, "get_tags") def test_gitee_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gitee"]) @@ -263,6 +285,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, WARNING) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GnuftpReleaseTags, "get_tags") def test_gnu_ftp_repo_success(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gnu-ftp"]) @@ -271,6 +294,7 @@ class TestCheckYamlRepo(unittest.TestCase): result = self.cy.check_repo() self.assertEqual(result, SUCCESS) + @unittest.skipIf((not ACCESS2INTERNET), "skip testcase need to access internet") @mock.patch.object(GnuftpReleaseTags, "get_tags") def test_gnu_ftp_repo_failed(self, mock_get_tags): self.cy.set_yaml(self.TEST_YAML_DIR["gnu-ftp"])