From cab86e849237927ceabc00c7dd51d04dafd9b1ef Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Mon, 19 Sep 2022 08:12:19 +0000 Subject: [PATCH 01/10] update src/lib/trigger.sh. Signed-off-by: liangchengh <1242747392@qq.com> --- src/ac/acl/source_consistency/__init__.py | 0 .../source_consistency/check_consistency.py | 233 ++++++++++++++++++ src/ac/framework/ac.yaml | 11 +- src/lib/trigger.sh | 2 + 4 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 src/ac/acl/source_consistency/__init__.py create mode 100644 src/ac/acl/source_consistency/check_consistency.py diff --git a/src/ac/acl/source_consistency/__init__.py b/src/ac/acl/source_consistency/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py new file mode 100644 index 0000000..606b88a --- /dev/null +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -0,0 +1,233 @@ +import hashlib +import logging +import os +import re +import shutil +import sqlite3 +from sqlite3 import Error + +from src.ac.common.gitee_repo import GiteeRepo +from src.ac.framework.ac_base import BaseCheck +from src.ac.framework.ac_result import FAILED, SUCCESS, WARNING + +logger = logging.getLogger("ac") + + +class CheckSourceConsistency(BaseCheck): + """ + check source consistence + """ + + def __init__(self, workspace, repo, conf=None): + super(CheckSourceConsistency, self).__init__(workspace, repo, conf) + + self._work_dir = os.path.join(workspace, "source_consistency") + shutil.copytree(os.path.join(workspace, repo), self._work_dir) + self._repo = repo + self.TEMP_TXT_PATH = os.path.join(self._work_dir, "temp.txt") + self.rpmbuild_dir = os.path.join(workspace, "rpmbuild") + self.RPMBUILD_BUILD_PATH = os.path.join(self.rpmbuild_dir, "BUILD") + self.RPMBUILD_SOURCE_PATH = os.path.join(self.rpmbuild_dir, "SOURCE") + self.database_path = "source_clean.db" + + def __call__(self, *args, **kwargs): + """ + 入口函数 + :param args: + :param kwargs: + :return: + """ + logger.info("check %s source consistency ...", self._repo) + _ = not os.path.exists("log") and os.mkdir("log") + try: + return self.start_check_with_order("source_consistency") + finally: + self.clear_temp() + + def check_source_consistency(self): + """ + 检查源码包是否一致 + """ + self.create_rpmbuild_dir() + source_url = self.get_source_url() + if source_url == "": + logger.warning("no valid source url") + return SUCCESS + + package_name = self.get_package_from_source(source_url) + if package_name not in os.listdir(self._work_dir): + logger.warning("no source package file") + return SUCCESS + + native_sha256sum = self.get_sha256sum(os.path.join(self._work_dir, package_name)) + if native_sha256sum == "": + logger.warning("get sha256sum of native source package failed") + return SUCCESS + + con = self.create_connection() + remote_sha256sum = self.get_sha256sum_from_url(con, source_url) + if remote_sha256sum == "": + logger.warning("no url in source_clean.db") + return SUCCESS + if native_sha256sum != remote_sha256sum: + logger.error("repo is inconsistency") + return FAILED + + return SUCCESS + + def create_rpmbuild_dir(self): + """ + 创建临时rpmbuild目录 + """ + os.mkdir(self.rpmbuild_dir) + os.mkdir(os.path.join(self.rpmbuild_dir, "BUILD")) + os.mkdir(os.path.join(self.rpmbuild_dir, "BUILDROOT")) + os.mkdir(os.path.join(self.rpmbuild_dir, "RPMS")) + os.mkdir(os.path.join(self.rpmbuild_dir, "SOURCE")) + os.mkdir(os.path.join(self.rpmbuild_dir, "SPECS")) + os.mkdir(os.path.join(self.rpmbuild_dir, "SRPMS")) + + def clear_temp(self): + """ + 清理生成的中间文件 + """ + if os.path.exists(self._work_dir): + shutil.rmtree(self._work_dir) + try: + files_list = os.listdir(self.RPMBUILD_SOURCE_PATH) + for file in files_list: + os.remove(os.path.join(self.RPMBUILD_SOURCE_PATH, file)) + files_list = os.listdir(self.RPMBUILD_BUILD_PATH) + for file in files_list: + os.remove(os.path.join(self.RPMBUILD_BUILD_PATH, file)) + except Exception as e: + logger.warning(e) + + def create_connection(self): + """ + 与数据库建立连接 + """ + logger.info("getting connection with source_clean.db ...") + try: + con = sqlite3.connect(self.database_path) + return con + except Error: + logger.error(Error) + + @staticmethod + def get_sha256sum_from_url(con, url): + """ + 查询数据库，获取url的sha256sum值 + """ + logger.info("getting sha256sum of remote source package from source_clean.db ...") + cursor_obj = con.cursor() + cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) + rows = cursor_obj.fetchall() + for row in rows: + return row[0] + return "" + + @staticmethod + def get_sha256sum(package): + """ + 计算文件的sha256sum值 + """ + logger.info("getting sha256sum of native source package...") + native_sha256sum = "" + try: + with open(package, "rb") as f: + sha256obj = hashlib.sha256() + sha256obj.update(f.read()) + native_sha256sum = sha256obj.hexdigest() + except Exception as e: + logger.warning(e) + if native_sha256sum == "": + try: + native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] + except Exception as e: + logger.warning(e) + return native_sha256sum.strip() + + @staticmethod + def get_package_from_source(url): + """ + 从url中获取包名 + """ + package_name = url.split("/")[-1].strip() + return package_name + + def get_source_url(self): + """ + 获取spec文件中的Source URL + """ + files_list = os.listdir(self._work_dir) + if len(files_list) == 0: + logger.error("compressed fail") + return "" + if self._repo + ".spec" not in files_list: + logger.error("no spec file") + return "" + source_url = self.get_source_from_rpmbuild() + return source_url + + def get_source_from_rpmbuild(self): + """ + rpmbuild解析出可查询的Source URL + """ + spec_file = os.path.join(self._work_dir, self._repo + ".spec") + self.spec_read(spec_file) + source_url = self.do_rpmbuild() + return source_url + + def spec_read(self, spec_file): + """ + 读取spec文件并生成新的spec文件 + """ + logger.info("reading spec file : %s ...", os.path.basename(spec_file)) + + new_spec_file = os.path.join(self.RPMBUILD_SOURCE_PATH, "get_source.spec") + cond_source = re.compile("^Source0*") + source_url = "" + new_spec_content = "" + for line in open(spec_file): + line = line.strip() + if line.startswith("%prep"): + break + elif cond_source.match(line) or re.match("^Source.*", line): + if source_url == "": + if ":" in line: + line_list = line.split(":") + source_url = ":".join(line_list[1:]).strip() + new_spec_content += line + os.linesep + new_spec_content += self.get_prep_function(source_url) + logger.info("generating new spec file ...") + f = open(new_spec_file, "w") + f.write(new_spec_content) + f.close() + + def get_prep_function(self, url): + """ + 生成spec文件%prep部分的内容 + """ + logger.info("generating %prep function") + function_content = "%prep" + os.linesep + function_content += "source={0}".format(url) + os.linesep + function_content += "cd {0}".format(self.RPMBUILD_SOURCE_PATH) + os.linesep + function_content += "echo $source > {0}".format(self.TEMP_TXT_PATH) + os.linesep + return function_content + + def do_rpmbuild(self): + """ + 对新生成的spec文件执行rpmbuild + """ + logger.info("start to do rpmbuild") + new_spec_file = os.path.join(self.RPMBUILD_SOURCE_PATH, "get_source.spec") + res = os.system("rpmbuild -bp --nodeps {0} --define \"_topdir {1}\"".format(new_spec_file, self.rpmbuild_dir)) + if res != 0: + logger.error("do rpmbuild fail") + if not os.path.exists(self.TEMP_TXT_PATH): + return "" + f = open(self.TEMP_TXT_PATH, "r") + source_url = f.read().strip() + f.close() + return source_url diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 4d7bc70..f96be37 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -29,6 +29,10 @@ src-openeuler: exclude: True commit_msg: exclude: True + source_consistency: + hint: check_consistency + module: source_consistency.check_consistency + entry: CheckSourceConsistency openeuler: spec: exclude: True @@ -46,7 +50,7 @@ openeuler: hint: check_sca module: sca.check_sca entry: CheckSCA - deny_list: ["bishengjdk-17", "bishengjdk-8", "bishengjdk-11", "bishengjdk-riscv", "gcc", "kernel"] + deny_list: [] openlibing: hint: code module: openlibing.check_code @@ -58,10 +62,11 @@ openeuler: "isula-build", "cve-manager", "attest-tools", "oec-hardware", "itrustee_sdk", "wisdom-advisor", "isula-transform", " itrustee_client", "A-Tune-Collector", "itrustee_tzdriver", "website-v2", "yocto-poky", "bishengjdk-17", "bishengjdk-8", "bishengjdk-11", "bishengjdk-riscv", "powerapi", - "eagle", "dcs", "astream", "QARobot", "oec-application", "gala-gopher", "gala-anteater", - "gala-spider", "gala-ragdoll"] + "eagle", "dcs", "astream", "QARobot", "oec-application"] commit_msg: hint: commit_msg module: commit_msg.check_commit_msg entry: CheckCommitMsg allow_list: ["yocto-meta-openeuler"] + source_consistency: + exclude: True diff --git a/src/lib/trigger.sh b/src/lib/trigger.sh index 6a8585f..ccbc8eb 100644 --- a/src/lib/trigger.sh +++ b/src/lib/trigger.sh @@ -26,6 +26,8 @@ function clearn_env() { EOF ) ssh -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR root@${repo_server} "$remote_dir_reset_cmd" + log_info "***** Start to copy db file *****" + scp -r -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@${repo_server}:/repo/soe/sql/source_clean.db . || log_info "file source_clean not exist" } # 开始下载kernel代码 -- Gitee From c2335ed4098abf5a2969fcdef5b85146ecbb9eec Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Tue, 20 Sep 2022 10:36:17 +0800 Subject: [PATCH 02/10] "repair check_code problem" --- .../source_consistency/check_consistency.py | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 606b88a..b8aee55 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -2,6 +2,7 @@ import hashlib import logging import os import re +import stat import shutil import sqlite3 from sqlite3 import Error @@ -24,10 +25,10 @@ class CheckSourceConsistency(BaseCheck): self._work_dir = os.path.join(workspace, "source_consistency") shutil.copytree(os.path.join(workspace, repo), self._work_dir) self._repo = repo - self.TEMP_TXT_PATH = os.path.join(self._work_dir, "temp.txt") + self.temp_txt_ppath = os.path.join(self._work_dir, "temp.txt") self.rpmbuild_dir = os.path.join(workspace, "rpmbuild") - self.RPMBUILD_BUILD_PATH = os.path.join(self.rpmbuild_dir, "BUILD") - self.RPMBUILD_SOURCE_PATH = os.path.join(self.rpmbuild_dir, "SOURCE") + self.rpmbuild_build_path = os.path.join(self.rpmbuild_dir, "BUILD") + self.rpmbuild_source_path = os.path.join(self.rpmbuild_dir, "SOURCE") self.database_path = "source_clean.db" def __call__(self, *args, **kwargs): @@ -93,26 +94,12 @@ class CheckSourceConsistency(BaseCheck): """ if os.path.exists(self._work_dir): shutil.rmtree(self._work_dir) - try: - files_list = os.listdir(self.RPMBUILD_SOURCE_PATH) - for file in files_list: - os.remove(os.path.join(self.RPMBUILD_SOURCE_PATH, file)) - files_list = os.listdir(self.RPMBUILD_BUILD_PATH) - for file in files_list: - os.remove(os.path.join(self.RPMBUILD_BUILD_PATH, file)) - except Exception as e: - logger.warning(e) - - def create_connection(self): - """ - 与数据库建立连接 - """ - logger.info("getting connection with source_clean.db ...") - try: - con = sqlite3.connect(self.database_path) - return con - except Error: - logger.error(Error) + files_list = os.listdir(self.rpmbuild_source_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_source_path, file)) + files_list = os.listdir(self.rpmbuild_build_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_build_path, file)) @staticmethod def get_sha256sum_from_url(con, url): @@ -127,6 +114,18 @@ class CheckSourceConsistency(BaseCheck): return row[0] return "" + def create_connection(self): + """ + 与数据库建立连接 + """ + logger.info("getting connection with source_clean.db ...") + try: + con = sqlite3.connect(self.database_path) + return con + except Error: + logger.error(Error) + return True + @staticmethod def get_sha256sum(package): """ @@ -185,7 +184,7 @@ class CheckSourceConsistency(BaseCheck): """ logger.info("reading spec file : %s ...", os.path.basename(spec_file)) - new_spec_file = os.path.join(self.RPMBUILD_SOURCE_PATH, "get_source.spec") + new_spec_file = os.path.join(self.rpmbuild_source_path, "get_source.spec") cond_source = re.compile("^Source0*") source_url = "" new_spec_content = "" @@ -201,9 +200,10 @@ class CheckSourceConsistency(BaseCheck): new_spec_content += line + os.linesep new_spec_content += self.get_prep_function(source_url) logger.info("generating new spec file ...") - f = open(new_spec_file, "w") - f.write(new_spec_content) - f.close() + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + modes = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(new_spec_file, flags, modes), 'w') as f: + f.write(new_spec_content) def get_prep_function(self, url): """ @@ -212,8 +212,8 @@ class CheckSourceConsistency(BaseCheck): logger.info("generating %prep function") function_content = "%prep" + os.linesep function_content += "source={0}".format(url) + os.linesep - function_content += "cd {0}".format(self.RPMBUILD_SOURCE_PATH) + os.linesep - function_content += "echo $source > {0}".format(self.TEMP_TXT_PATH) + os.linesep + function_content += "cd {0}".format(self.rpmbuild_source_path) + os.linesep + function_content += "echo $source > {0}".format(self.temp_txt_ppath) + os.linesep return function_content def do_rpmbuild(self): @@ -221,13 +221,13 @@ class CheckSourceConsistency(BaseCheck): 对新生成的spec文件执行rpmbuild """ logger.info("start to do rpmbuild") - new_spec_file = os.path.join(self.RPMBUILD_SOURCE_PATH, "get_source.spec") + new_spec_file = os.path.join(self.rpmbuild_source_path, "get_source.spec") res = os.system("rpmbuild -bp --nodeps {0} --define \"_topdir {1}\"".format(new_spec_file, self.rpmbuild_dir)) if res != 0: logger.error("do rpmbuild fail") - if not os.path.exists(self.TEMP_TXT_PATH): + if not os.path.exists(self.temp_txt_ppath): return "" - f = open(self.TEMP_TXT_PATH, "r") + f = open(self.temp_txt_ppath, "r") source_url = f.read().strip() f.close() return source_url -- Gitee From 822d1d0dc48a31918ce1296fa08e750090316c7a Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Tue, 20 Sep 2022 10:53:33 +0800 Subject: [PATCH 03/10] "adjust def order" --- .../source_consistency/check_consistency.py | 134 +++++++++--------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index b8aee55..b634e23 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -88,73 +88,6 @@ class CheckSourceConsistency(BaseCheck): os.mkdir(os.path.join(self.rpmbuild_dir, "SPECS")) os.mkdir(os.path.join(self.rpmbuild_dir, "SRPMS")) - def clear_temp(self): - """ - 清理生成的中间文件 - """ - if os.path.exists(self._work_dir): - shutil.rmtree(self._work_dir) - files_list = os.listdir(self.rpmbuild_source_path) - for file in files_list: - os.remove(os.path.join(self.rpmbuild_source_path, file)) - files_list = os.listdir(self.rpmbuild_build_path) - for file in files_list: - os.remove(os.path.join(self.rpmbuild_build_path, file)) - - @staticmethod - def get_sha256sum_from_url(con, url): - """ - 查询数据库，获取url的sha256sum值 - """ - logger.info("getting sha256sum of remote source package from source_clean.db ...") - cursor_obj = con.cursor() - cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) - rows = cursor_obj.fetchall() - for row in rows: - return row[0] - return "" - - def create_connection(self): - """ - 与数据库建立连接 - """ - logger.info("getting connection with source_clean.db ...") - try: - con = sqlite3.connect(self.database_path) - return con - except Error: - logger.error(Error) - return True - - @staticmethod - def get_sha256sum(package): - """ - 计算文件的sha256sum值 - """ - logger.info("getting sha256sum of native source package...") - native_sha256sum = "" - try: - with open(package, "rb") as f: - sha256obj = hashlib.sha256() - sha256obj.update(f.read()) - native_sha256sum = sha256obj.hexdigest() - except Exception as e: - logger.warning(e) - if native_sha256sum == "": - try: - native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] - except Exception as e: - logger.warning(e) - return native_sha256sum.strip() - - @staticmethod - def get_package_from_source(url): - """ - 从url中获取包名 - """ - package_name = url.split("/")[-1].strip() - return package_name - def get_source_url(self): """ 获取spec文件中的Source URL @@ -231,3 +164,70 @@ class CheckSourceConsistency(BaseCheck): source_url = f.read().strip() f.close() return source_url + + @staticmethod + def get_package_from_source(url): + """ + 从url中获取包名 + """ + package_name = url.split("/")[-1].strip() + return package_name + + @staticmethod + def get_sha256sum(package): + """ + 计算文件的sha256sum值 + """ + logger.info("getting sha256sum of native source package...") + native_sha256sum = "" + try: + with open(package, "rb") as f: + sha256obj = hashlib.sha256() + sha256obj.update(f.read()) + native_sha256sum = sha256obj.hexdigest() + except Exception as e: + logger.warning(e) + if native_sha256sum == "": + try: + native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] + except Exception as e: + logger.warning(e) + return native_sha256sum.strip() + + def create_connection(self): + """ + 与数据库建立连接 + """ + logger.info("getting connection with source_clean.db ...") + try: + con = sqlite3.connect(self.database_path) + return con + except Error: + logger.error(Error) + return True + + @staticmethod + def get_sha256sum_from_url(con, url): + """ + 查询数据库，获取url的sha256sum值 + """ + logger.info("getting sha256sum of remote source package from source_clean.db ...") + cursor_obj = con.cursor() + cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) + rows = cursor_obj.fetchall() + for row in rows: + return row[0] + return "" + + def clear_temp(self): + """ + 清理生成的中间文件 + """ + if os.path.exists(self._work_dir): + shutil.rmtree(self._work_dir) + files_list = os.listdir(self.rpmbuild_source_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_source_path, file)) + files_list = os.listdir(self.rpmbuild_build_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_build_path, file)) -- Gitee From 6cf4411ffcf165fd369ebcdc1ceb6edb0d2e2cb8 Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Tue, 20 Sep 2022 11:12:37 +0800 Subject: [PATCH 04/10] "def order" --- .../source_consistency/check_consistency.py | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index b634e23..511808f 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -45,6 +45,48 @@ class CheckSourceConsistency(BaseCheck): finally: self.clear_temp() + @staticmethod + def get_package_from_source(url): + """ + 从url中获取包名 + """ + package_name = url.split("/")[-1].strip() + return package_name + + @staticmethod + def get_sha256sum(package): + """ + 计算文件的sha256sum值 + """ + logger.info("getting sha256sum of native source package...") + native_sha256sum = "" + try: + with open(package, "rb") as f: + sha256obj = hashlib.sha256() + sha256obj.update(f.read()) + native_sha256sum = sha256obj.hexdigest() + except Exception as e: + logger.warning(e) + if native_sha256sum == "": + try: + native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] + except Exception as e: + logger.warning(e) + return native_sha256sum.strip() + + @staticmethod + def get_sha256sum_from_url(con, url): + """ + 查询数据库，获取url的sha256sum值 + """ + logger.info("getting sha256sum of remote source package from source_clean.db ...") + cursor_obj = con.cursor() + cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) + rows = cursor_obj.fetchall() + for row in rows: + return row[0] + return "" + def check_source_consistency(self): """ 检查源码包是否一致 @@ -165,35 +207,6 @@ class CheckSourceConsistency(BaseCheck): f.close() return source_url - @staticmethod - def get_package_from_source(url): - """ - 从url中获取包名 - """ - package_name = url.split("/")[-1].strip() - return package_name - - @staticmethod - def get_sha256sum(package): - """ - 计算文件的sha256sum值 - """ - logger.info("getting sha256sum of native source package...") - native_sha256sum = "" - try: - with open(package, "rb") as f: - sha256obj = hashlib.sha256() - sha256obj.update(f.read()) - native_sha256sum = sha256obj.hexdigest() - except Exception as e: - logger.warning(e) - if native_sha256sum == "": - try: - native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] - except Exception as e: - logger.warning(e) - return native_sha256sum.strip() - def create_connection(self): """ 与数据库建立连接 @@ -206,19 +219,6 @@ class CheckSourceConsistency(BaseCheck): logger.error(Error) return True - @staticmethod - def get_sha256sum_from_url(con, url): - """ - 查询数据库，获取url的sha256sum值 - """ - logger.info("getting sha256sum of remote source package from source_clean.db ...") - cursor_obj = con.cursor() - cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) - rows = cursor_obj.fetchall() - for row in rows: - return row[0] - return "" - def clear_temp(self): """ 清理生成的中间文件 -- Gitee From d44b1aad95c34dcb03bc149858824124f547c7e1 Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Tue, 20 Sep 2022 11:29:12 +0800 Subject: [PATCH 05/10] "sync code" --- src/ac/acl/spec/check_spec.py | 7 ++++++- src/ac/framework/ac.yaml | 7 ++++--- src/utils/compare_package.py | 2 +- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index cdf96a2..e8eb54e 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -320,7 +320,12 @@ class CheckSpec(BaseCheck): logger.error("%s release or version incorrect format,please keep it consistent: version-release \n" "e.g: 1.0.0-1", changelog_con) return False - version_num, release_num = obj_s.group(0).split("-") + try: + version_num, release_num = obj_s.group(0).split("-") + except ValueError as e: + logger.error("%s release or version incorrect format,please keep it consistent: version-release \n" + "e.g: 1.0.0-1", changelog_con) + return False if version_num != version: logger.error("version error in changelog: %s is different from %s", version_num, version) return False diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index f96be37..658710a 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -50,7 +50,7 @@ openeuler: hint: check_sca module: sca.check_sca entry: CheckSCA - deny_list: [] + deny_list: ["bishengjdk-17", "bishengjdk-8", "bishengjdk-11", "bishengjdk-riscv", "gcc", "kernel"] openlibing: hint: code module: openlibing.check_code @@ -62,11 +62,12 @@ openeuler: "isula-build", "cve-manager", "attest-tools", "oec-hardware", "itrustee_sdk", "wisdom-advisor", "isula-transform", " itrustee_client", "A-Tune-Collector", "itrustee_tzdriver", "website-v2", "yocto-poky", "bishengjdk-17", "bishengjdk-8", "bishengjdk-11", "bishengjdk-riscv", "powerapi", - "eagle", "dcs", "astream", "QARobot", "oec-application"] + "eagle", "dcs", "astream", "QARobot", "oec-application", "gala-gopher", "gala-anteater", + "gala-spider", "gala-ragdoll"] commit_msg: hint: commit_msg module: commit_msg.check_commit_msg entry: CheckCommitMsg allow_list: ["yocto-meta-openeuler"] source_consistency: - exclude: True + exclude: True \ No newline at end of file diff --git a/src/utils/compare_package.py b/src/utils/compare_package.py index 7f992b0..2ed5945 100644 --- a/src/utils/compare_package.py +++ b/src/utils/compare_package.py @@ -34,7 +34,7 @@ class ComparePackage(object): MIN_COLUMN_WIDTH = 16 MAX_TABLE_WIDTH = 150 - all_check_item = ["rpm abi", "rpm kabi", "drive kabi", "rpm jabi", "rpm config", "rpm service", + all_check_item = ["rpm abi", "rpm kabi", "drive kabi", "rpm jabi", "rpm config", "rpm service", "service detail", "rpm kconfig", "rpm provides", "rpm requires", "rpm files", "rpm cmd", "rpm header", "rpm lib"] def __init__(self, logger): -- Gitee From 9d4e6082dec71a96e7ba324a1d1c99a78d85d807 Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Tue, 20 Sep 2022 13:58:46 +0800 Subject: [PATCH 06/10] "solve some problem" --- .../source_consistency/check_consistency.py | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 511808f..7c456b2 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -109,6 +109,7 @@ class CheckSourceConsistency(BaseCheck): con = self.create_connection() remote_sha256sum = self.get_sha256sum_from_url(con, source_url) + con.close() if remote_sha256sum == "": logger.warning("no url in source_clean.db") return SUCCESS @@ -136,7 +137,7 @@ class CheckSourceConsistency(BaseCheck): """ files_list = os.listdir(self._work_dir) if len(files_list) == 0: - logger.error("compressed fail") + logger.error("copy repo error") return "" if self._repo + ".spec" not in files_list: logger.error("no spec file") @@ -163,16 +164,17 @@ class CheckSourceConsistency(BaseCheck): cond_source = re.compile("^Source0*") source_url = "" new_spec_content = "" - for line in open(spec_file): - line = line.strip() - if line.startswith("%prep"): - break - elif cond_source.match(line) or re.match("^Source.*", line): - if source_url == "": - if ":" in line: - line_list = line.split(":") - source_url = ":".join(line_list[1:]).strip() - new_spec_content += line + os.linesep + with open(spec_file) as f: + for line in f: + line = line.strip() + if line.startswith("%prep"): + break + elif cond_source.match(line) or re.match("^Source.*", line): + if source_url == "": + if ":" in line: + line_list = line.split(":") + source_url = ":".join(line_list[1:]).strip() + new_spec_content += line + os.linesep new_spec_content += self.get_prep_function(source_url) logger.info("generating new spec file ...") flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL @@ -213,8 +215,9 @@ class CheckSourceConsistency(BaseCheck): """ logger.info("getting connection with source_clean.db ...") try: - con = sqlite3.connect(self.database_path) - return con + if os.path.exists(self.database_path): + con = sqlite3.connect(self.database_path) + return con except Error: logger.error(Error) return True -- Gitee From 17c0c1d3318f94efd317aa100222e0e40d5e0555 Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Wed, 21 Sep 2022 02:29:47 +0000 Subject: [PATCH 07/10] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=BA=93=E8=BF=9E=E6=8E=A5=E5=AF=B9=E8=B1=A1=E5=A4=84=E7=90=86?= =?UTF-8?q?=E3=80=81=E6=97=A5=E5=BF=97=E6=8F=90=E7=A4=BA=E7=8A=B6=E6=80=81?= =?UTF-8?q?=E5=8F=8A=E4=BB=A3=E7=A0=81=E8=A7=84=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: liangchengh <1242747392@qq.com> --- .../source_consistency/check_consistency.py | 89 ++++++++----------- 1 file changed, 37 insertions(+), 52 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 7c456b2..6708fe6 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -25,11 +25,12 @@ class CheckSourceConsistency(BaseCheck): self._work_dir = os.path.join(workspace, "source_consistency") shutil.copytree(os.path.join(workspace, repo), self._work_dir) self._repo = repo - self.temp_txt_ppath = os.path.join(self._work_dir, "temp.txt") + self.temp_txt_path = os.path.join(self._work_dir, "temp.txt") self.rpmbuild_dir = os.path.join(workspace, "rpmbuild") self.rpmbuild_build_path = os.path.join(self.rpmbuild_dir, "BUILD") - self.rpmbuild_source_path = os.path.join(self.rpmbuild_dir, "SOURCE") + self.rpmbuild_sources_path = os.path.join(self.rpmbuild_dir, "SOURCES") self.database_path = "source_clean.db" + self.con = self.create_connection() def __call__(self, *args, **kwargs): """ @@ -43,6 +44,8 @@ class CheckSourceConsistency(BaseCheck): try: return self.start_check_with_order("source_consistency") finally: + if self.con is not None: + self.con.close() self.clear_temp() @staticmethod @@ -74,63 +77,36 @@ class CheckSourceConsistency(BaseCheck): logger.warning(e) return native_sha256sum.strip() - @staticmethod - def get_sha256sum_from_url(con, url): - """ - 查询数据库，获取url的sha256sum值 - """ - logger.info("getting sha256sum of remote source package from source_clean.db ...") - cursor_obj = con.cursor() - cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) - rows = cursor_obj.fetchall() - for row in rows: - return row[0] - return "" - def check_source_consistency(self): """ 检查源码包是否一致 """ - self.create_rpmbuild_dir() + os.makedirs(os.path.join(self.rpmbuild_dir, "SOURCES"), exist_ok=True) source_url = self.get_source_url() if source_url == "": logger.warning("no valid source url") - return SUCCESS + return WARNING package_name = self.get_package_from_source(source_url) if package_name not in os.listdir(self._work_dir): logger.warning("no source package file") - return SUCCESS + return WARNING native_sha256sum = self.get_sha256sum(os.path.join(self._work_dir, package_name)) if native_sha256sum == "": logger.warning("get sha256sum of native source package failed") - return SUCCESS + return WARNING - con = self.create_connection() - remote_sha256sum = self.get_sha256sum_from_url(con, source_url) - con.close() + remote_sha256sum = self.get_sha256sum_from_url(source_url) if remote_sha256sum == "": logger.warning("no url in source_clean.db") - return SUCCESS + return WARNING if native_sha256sum != remote_sha256sum: logger.error("repo is inconsistency") return FAILED return SUCCESS - def create_rpmbuild_dir(self): - """ - 创建临时rpmbuild目录 - """ - os.mkdir(self.rpmbuild_dir) - os.mkdir(os.path.join(self.rpmbuild_dir, "BUILD")) - os.mkdir(os.path.join(self.rpmbuild_dir, "BUILDROOT")) - os.mkdir(os.path.join(self.rpmbuild_dir, "RPMS")) - os.mkdir(os.path.join(self.rpmbuild_dir, "SOURCE")) - os.mkdir(os.path.join(self.rpmbuild_dir, "SPECS")) - os.mkdir(os.path.join(self.rpmbuild_dir, "SRPMS")) - def get_source_url(self): """ 获取spec文件中的Source URL @@ -150,17 +126,17 @@ class CheckSourceConsistency(BaseCheck): rpmbuild解析出可查询的Source URL """ spec_file = os.path.join(self._work_dir, self._repo + ".spec") - self.spec_read(spec_file) + self.generate_new_spec(spec_file) source_url = self.do_rpmbuild() return source_url - def spec_read(self, spec_file): + def generate_new_spec(self, spec_file): """ 读取spec文件并生成新的spec文件 """ logger.info("reading spec file : %s ...", os.path.basename(spec_file)) - new_spec_file = os.path.join(self.rpmbuild_source_path, "get_source.spec") + new_spec_file = os.path.join(self.rpmbuild_sources_path, "get_source.spec") cond_source = re.compile("^Source0*") source_url = "" new_spec_content = "" @@ -172,8 +148,7 @@ class CheckSourceConsistency(BaseCheck): elif cond_source.match(line) or re.match("^Source.*", line): if source_url == "": if ":" in line: - line_list = line.split(":") - source_url = ":".join(line_list[1:]).strip() + source_url = ":".join(line.split(":")[1:]).strip() new_spec_content += line + os.linesep new_spec_content += self.get_prep_function(source_url) logger.info("generating new spec file ...") @@ -189,8 +164,8 @@ class CheckSourceConsistency(BaseCheck): logger.info("generating %prep function") function_content = "%prep" + os.linesep function_content += "source={0}".format(url) + os.linesep - function_content += "cd {0}".format(self.rpmbuild_source_path) + os.linesep - function_content += "echo $source > {0}".format(self.temp_txt_ppath) + os.linesep + function_content += "cd {0}".format(self.rpmbuild_sources_path) + os.linesep + function_content += "echo $source > {0}".format(self.temp_txt_path) + os.linesep return function_content def do_rpmbuild(self): @@ -198,13 +173,13 @@ class CheckSourceConsistency(BaseCheck): 对新生成的spec文件执行rpmbuild """ logger.info("start to do rpmbuild") - new_spec_file = os.path.join(self.rpmbuild_source_path, "get_source.spec") + new_spec_file = os.path.join(self.rpmbuild_sources_path, "get_source.spec") res = os.system("rpmbuild -bp --nodeps {0} --define \"_topdir {1}\"".format(new_spec_file, self.rpmbuild_dir)) if res != 0: logger.error("do rpmbuild fail") - if not os.path.exists(self.temp_txt_ppath): + if not os.path.exists(self.temp_txt_path): return "" - f = open(self.temp_txt_ppath, "r") + f = open(self.temp_txt_path, "r") source_url = f.read().strip() f.close() return source_url @@ -220,7 +195,22 @@ class CheckSourceConsistency(BaseCheck): return con except Error: logger.error(Error) - return True + return None + + def get_sha256sum_from_url(self, url): + """ + 查询数据库，获取url的sha256sum值 + """ + logger.info("getting sha256sum of remote source package from source_clean.db ...") + if self.con is None: + logger.warning("failed to connect to database") + return "" + cursor_obj = self.con.cursor() + cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) + row = cursor_obj.fetchone() + if row: + return row[0] + return "" def clear_temp(self): """ @@ -228,9 +218,4 @@ class CheckSourceConsistency(BaseCheck): """ if os.path.exists(self._work_dir): shutil.rmtree(self._work_dir) - files_list = os.listdir(self.rpmbuild_source_path) - for file in files_list: - os.remove(os.path.join(self.rpmbuild_source_path, file)) - files_list = os.listdir(self.rpmbuild_build_path) - for file in files_list: - os.remove(os.path.join(self.rpmbuild_build_path, file)) + shutil.rmtree(self.rpmbuild_dir) -- Gitee From a3f9cd958e41de254f3f0588953cbd32466a05d7 Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Wed, 21 Sep 2022 02:52:13 +0000 Subject: [PATCH 08/10] =?UTF-8?q?=E9=80=82=E9=85=8D=E4=BB=93=E5=BA=93?= =?UTF-8?q?=E4=B8=ADspec=E6=96=87=E4=BB=B6=E5=90=8D=E4=B8=8E=E4=BB=93?= =?UTF-8?q?=E5=BA=93=E5=90=8D=E4=B8=8D=E4=B8=80=E8=87=B4=E7=9A=84=E6=83=85?= =?UTF-8?q?=E5=86=B5=EF=BC=9A=E6=9F=A5=E6=89=BEspec=E4=B8=BA=E6=8B=93?= =?UTF-8?q?=E5=B1=95=E5=90=8D=E7=9A=84=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: liangchengh <1242747392@qq.com> --- .../source_consistency/check_consistency.py | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 6708fe6..d9f0fb3 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -111,21 +111,30 @@ class CheckSourceConsistency(BaseCheck): """ 获取spec文件中的Source URL """ + spec_name = "" files_list = os.listdir(self._work_dir) if len(files_list) == 0: - logger.error("copy repo error") + logger.error("copy repo error, please check!") return "" if self._repo + ".spec" not in files_list: - logger.error("no spec file") - return "" - source_url = self.get_source_from_rpmbuild() + logger.warning("no such spec file: " + self._repo + ".spec") + for file_name in files_list: + if file_name.endswith(".spec"): + spec_name = file_name + break + if spec_name == "": + logger.error("no spec file, please check!") + return "" + source_url = self.get_source_from_rpmbuild(spec_name) return source_url - def get_source_from_rpmbuild(self): + def get_source_from_rpmbuild(self, spec_name=""): """ rpmbuild解析出可查询的Source URL """ - spec_file = os.path.join(self._work_dir, self._repo + ".spec") + if spec_name == "": + spec_name = self._repo + "spec" + spec_file = os.path.join(self._work_dir, spec_name) self.generate_new_spec(spec_file) source_url = self.do_rpmbuild() return source_url -- Gitee From a642d661b637a086a14f9e2f99144507f3e78052 Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Wed, 21 Sep 2022 03:45:38 +0000 Subject: [PATCH 09/10] =?UTF-8?q?=E6=92=A4=E9=94=80=E4=B8=8D=E5=9C=A8?= =?UTF-8?q?=E6=9C=AC=E6=AC=A1=E4=BF=AE=E6=94=B9=E7=9A=84=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: liangchengh <1242747392@qq.com> --- src/utils/compare_package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/compare_package.py b/src/utils/compare_package.py index 2ed5945..7f992b0 100644 --- a/src/utils/compare_package.py +++ b/src/utils/compare_package.py @@ -34,7 +34,7 @@ class ComparePackage(object): MIN_COLUMN_WIDTH = 16 MAX_TABLE_WIDTH = 150 - all_check_item = ["rpm abi", "rpm kabi", "drive kabi", "rpm jabi", "rpm config", "rpm service", "service detail", + all_check_item = ["rpm abi", "rpm kabi", "drive kabi", "rpm jabi", "rpm config", "rpm service", "rpm kconfig", "rpm provides", "rpm requires", "rpm files", "rpm cmd", "rpm header", "rpm lib"] def __init__(self, logger): -- Gitee From c3241acca96f1f0411f92258f293224c8b965f12 Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Wed, 21 Sep 2022 03:47:25 +0000 Subject: [PATCH 10/10] =?UTF-8?q?=E6=92=A4=E9=94=80=E4=B8=8D=E5=9C=A8?= =?UTF-8?q?=E6=9C=AC=E6=AC=A1=E4=BF=AE=E6=94=B9=E7=9A=84=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: liangchengh <1242747392@qq.com> --- src/ac/acl/spec/check_spec.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/ac/acl/spec/check_spec.py b/src/ac/acl/spec/check_spec.py index e8eb54e..cdf96a2 100644 --- a/src/ac/acl/spec/check_spec.py +++ b/src/ac/acl/spec/check_spec.py @@ -320,12 +320,7 @@ class CheckSpec(BaseCheck): logger.error("%s release or version incorrect format,please keep it consistent: version-release \n" "e.g: 1.0.0-1", changelog_con) return False - try: - version_num, release_num = obj_s.group(0).split("-") - except ValueError as e: - logger.error("%s release or version incorrect format,please keep it consistent: version-release \n" - "e.g: 1.0.0-1", changelog_con) - return False + version_num, release_num = obj_s.group(0).split("-") if version_num != version: logger.error("version error in changelog: %s is different from %s", version_num, version) return False -- Gitee