From cab86e849237927ceabc00c7dd51d04dafd9b1ef Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Mon, 19 Sep 2022 08:12:19 +0000 Subject: [PATCH 1/4] update src/lib/trigger.sh. Signed-off-by: liangchengh <1242747392@qq.com> --- src/ac/acl/source_consistency/__init__.py | 0 .../source_consistency/check_consistency.py | 233 ++++++++++++++++++ src/ac/framework/ac.yaml | 11 +- src/lib/trigger.sh | 2 + 4 files changed, 243 insertions(+), 3 deletions(-) create mode 100644 src/ac/acl/source_consistency/__init__.py create mode 100644 src/ac/acl/source_consistency/check_consistency.py diff --git a/src/ac/acl/source_consistency/__init__.py b/src/ac/acl/source_consistency/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py new file mode 100644 index 0000000..606b88a --- /dev/null +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -0,0 +1,233 @@ +import hashlib +import logging +import os +import re +import shutil +import sqlite3 +from sqlite3 import Error + +from src.ac.common.gitee_repo import GiteeRepo +from src.ac.framework.ac_base import BaseCheck +from src.ac.framework.ac_result import FAILED, SUCCESS, WARNING + +logger = logging.getLogger("ac") + + +class CheckSourceConsistency(BaseCheck): + """ + check source consistence + """ + + def __init__(self, workspace, repo, conf=None): + super(CheckSourceConsistency, self).__init__(workspace, repo, conf) + + self._work_dir = os.path.join(workspace, "source_consistency") + shutil.copytree(os.path.join(workspace, repo), self._work_dir) + self._repo = repo + self.TEMP_TXT_PATH = os.path.join(self._work_dir, "temp.txt") + self.rpmbuild_dir = os.path.join(workspace, "rpmbuild") + self.RPMBUILD_BUILD_PATH = os.path.join(self.rpmbuild_dir, "BUILD") + self.RPMBUILD_SOURCE_PATH = os.path.join(self.rpmbuild_dir, "SOURCE") + self.database_path = "source_clean.db" + + def __call__(self, *args, **kwargs): + """ + 入口函数 + :param args: + :param kwargs: + :return: + """ + logger.info("check %s source consistency ...", self._repo) + _ = not os.path.exists("log") and os.mkdir("log") + try: + return self.start_check_with_order("source_consistency") + finally: + self.clear_temp() + + def check_source_consistency(self): + """ + 检查源码包是否一致 + """ + self.create_rpmbuild_dir() + source_url = self.get_source_url() + if source_url == "": + logger.warning("no valid source url") + return SUCCESS + + package_name = self.get_package_from_source(source_url) + if package_name not in os.listdir(self._work_dir): + logger.warning("no source package file") + return SUCCESS + + native_sha256sum = self.get_sha256sum(os.path.join(self._work_dir, package_name)) + if native_sha256sum == "": + logger.warning("get sha256sum of native source package failed") + return SUCCESS + + con = self.create_connection() + remote_sha256sum = self.get_sha256sum_from_url(con, source_url) + if remote_sha256sum == "": + logger.warning("no url in source_clean.db") + return SUCCESS + if native_sha256sum != remote_sha256sum: + logger.error("repo is inconsistency") + return FAILED + + return SUCCESS + + def create_rpmbuild_dir(self): + """ + 创建临时rpmbuild目录 + """ + os.mkdir(self.rpmbuild_dir) + os.mkdir(os.path.join(self.rpmbuild_dir, "BUILD")) + os.mkdir(os.path.join(self.rpmbuild_dir, "BUILDROOT")) + os.mkdir(os.path.join(self.rpmbuild_dir, "RPMS")) + os.mkdir(os.path.join(self.rpmbuild_dir, "SOURCE")) + os.mkdir(os.path.join(self.rpmbuild_dir, "SPECS")) + os.mkdir(os.path.join(self.rpmbuild_dir, "SRPMS")) + + def clear_temp(self): + """ + 清理生成的中间文件 + """ + if os.path.exists(self._work_dir): + shutil.rmtree(self._work_dir) + try: + files_list = os.listdir(self.RPMBUILD_SOURCE_PATH) + for file in files_list: + os.remove(os.path.join(self.RPMBUILD_SOURCE_PATH, file)) + files_list = os.listdir(self.RPMBUILD_BUILD_PATH) + for file in files_list: + os.remove(os.path.join(self.RPMBUILD_BUILD_PATH, file)) + except Exception as e: + logger.warning(e) + + def create_connection(self): + """ + 与数据库建立连接 + """ + logger.info("getting connection with source_clean.db ...") + try: + con = sqlite3.connect(self.database_path) + return con + except Error: + logger.error(Error) + + @staticmethod + def get_sha256sum_from_url(con, url): + """ + 查询数据库，获取url的sha256sum值 + """ + logger.info("getting sha256sum of remote source package from source_clean.db ...") + cursor_obj = con.cursor() + cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) + rows = cursor_obj.fetchall() + for row in rows: + return row[0] + return "" + + @staticmethod + def get_sha256sum(package): + """ + 计算文件的sha256sum值 + """ + logger.info("getting sha256sum of native source package...") + native_sha256sum = "" + try: + with open(package, "rb") as f: + sha256obj = hashlib.sha256() + sha256obj.update(f.read()) + native_sha256sum = sha256obj.hexdigest() + except Exception as e: + logger.warning(e) + if native_sha256sum == "": + try: + native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] + except Exception as e: + logger.warning(e) + return native_sha256sum.strip() + + @staticmethod + def get_package_from_source(url): + """ + 从url中获取包名 + """ + package_name = url.split("/")[-1].strip() + return package_name + + def get_source_url(self): + """ + 获取spec文件中的Source URL + """ + files_list = os.listdir(self._work_dir) + if len(files_list) == 0: + logger.error("compressed fail") + return "" + if self._repo + ".spec" not in files_list: + logger.error("no spec file") + return "" + source_url = self.get_source_from_rpmbuild() + return source_url + + def get_source_from_rpmbuild(self): + """ + rpmbuild解析出可查询的Source URL + """ + spec_file = os.path.join(self._work_dir, self._repo + ".spec") + self.spec_read(spec_file) + source_url = self.do_rpmbuild() + return source_url + + def spec_read(self, spec_file): + """ + 读取spec文件并生成新的spec文件 + """ + logger.info("reading spec file : %s ...", os.path.basename(spec_file)) + + new_spec_file = os.path.join(self.RPMBUILD_SOURCE_PATH, "get_source.spec") + cond_source = re.compile("^Source0*") + source_url = "" + new_spec_content = "" + for line in open(spec_file): + line = line.strip() + if line.startswith("%prep"): + break + elif cond_source.match(line) or re.match("^Source.*", line): + if source_url == "": + if ":" in line: + line_list = line.split(":") + source_url = ":".join(line_list[1:]).strip() + new_spec_content += line + os.linesep + new_spec_content += self.get_prep_function(source_url) + logger.info("generating new spec file ...") + f = open(new_spec_file, "w") + f.write(new_spec_content) + f.close() + + def get_prep_function(self, url): + """ + 生成spec文件%prep部分的内容 + """ + logger.info("generating %prep function") + function_content = "%prep" + os.linesep + function_content += "source={0}".format(url) + os.linesep + function_content += "cd {0}".format(self.RPMBUILD_SOURCE_PATH) + os.linesep + function_content += "echo $source > {0}".format(self.TEMP_TXT_PATH) + os.linesep + return function_content + + def do_rpmbuild(self): + """ + 对新生成的spec文件执行rpmbuild + """ + logger.info("start to do rpmbuild") + new_spec_file = os.path.join(self.RPMBUILD_SOURCE_PATH, "get_source.spec") + res = os.system("rpmbuild -bp --nodeps {0} --define \"_topdir {1}\"".format(new_spec_file, self.rpmbuild_dir)) + if res != 0: + logger.error("do rpmbuild fail") + if not os.path.exists(self.TEMP_TXT_PATH): + return "" + f = open(self.TEMP_TXT_PATH, "r") + source_url = f.read().strip() + f.close() + return source_url diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 4d7bc70..f96be37 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -29,6 +29,10 @@ src-openeuler: exclude: True commit_msg: exclude: True + source_consistency: + hint: check_consistency + module: source_consistency.check_consistency + entry: CheckSourceConsistency openeuler: spec: exclude: True @@ -46,7 +50,7 @@ openeuler: hint: check_sca module: sca.check_sca entry: CheckSCA - deny_list: ["bishengjdk-17", "bishengjdk-8", "bishengjdk-11", "bishengjdk-riscv", "gcc", "kernel"] + deny_list: [] openlibing: hint: code module: openlibing.check_code @@ -58,10 +62,11 @@ openeuler: "isula-build", "cve-manager", "attest-tools", "oec-hardware", "itrustee_sdk", "wisdom-advisor", "isula-transform", " itrustee_client", "A-Tune-Collector", "itrustee_tzdriver", "website-v2", "yocto-poky", "bishengjdk-17", "bishengjdk-8", "bishengjdk-11", "bishengjdk-riscv", "powerapi", - "eagle", "dcs", "astream", "QARobot", "oec-application", "gala-gopher", "gala-anteater", - "gala-spider", "gala-ragdoll"] + "eagle", "dcs", "astream", "QARobot", "oec-application"] commit_msg: hint: commit_msg module: commit_msg.check_commit_msg entry: CheckCommitMsg allow_list: ["yocto-meta-openeuler"] + source_consistency: + exclude: True diff --git a/src/lib/trigger.sh b/src/lib/trigger.sh index 6a8585f..ccbc8eb 100644 --- a/src/lib/trigger.sh +++ b/src/lib/trigger.sh @@ -26,6 +26,8 @@ function clearn_env() { EOF ) ssh -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR root@${repo_server} "$remote_dir_reset_cmd" + log_info "***** Start to copy db file *****" + scp -r -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@${repo_server}:/repo/soe/sql/source_clean.db . || log_info "file source_clean not exist" } # 开始下载kernel代码 -- Gitee From c2335ed4098abf5a2969fcdef5b85146ecbb9eec Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Tue, 20 Sep 2022 10:36:17 +0800 Subject: [PATCH 2/4] "repair check_code problem" --- .../source_consistency/check_consistency.py | 64 +++++++++---------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 606b88a..b8aee55 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -2,6 +2,7 @@ import hashlib import logging import os import re +import stat import shutil import sqlite3 from sqlite3 import Error @@ -24,10 +25,10 @@ class CheckSourceConsistency(BaseCheck): self._work_dir = os.path.join(workspace, "source_consistency") shutil.copytree(os.path.join(workspace, repo), self._work_dir) self._repo = repo - self.TEMP_TXT_PATH = os.path.join(self._work_dir, "temp.txt") + self.temp_txt_ppath = os.path.join(self._work_dir, "temp.txt") self.rpmbuild_dir = os.path.join(workspace, "rpmbuild") - self.RPMBUILD_BUILD_PATH = os.path.join(self.rpmbuild_dir, "BUILD") - self.RPMBUILD_SOURCE_PATH = os.path.join(self.rpmbuild_dir, "SOURCE") + self.rpmbuild_build_path = os.path.join(self.rpmbuild_dir, "BUILD") + self.rpmbuild_source_path = os.path.join(self.rpmbuild_dir, "SOURCE") self.database_path = "source_clean.db" def __call__(self, *args, **kwargs): @@ -93,26 +94,12 @@ class CheckSourceConsistency(BaseCheck): """ if os.path.exists(self._work_dir): shutil.rmtree(self._work_dir) - try: - files_list = os.listdir(self.RPMBUILD_SOURCE_PATH) - for file in files_list: - os.remove(os.path.join(self.RPMBUILD_SOURCE_PATH, file)) - files_list = os.listdir(self.RPMBUILD_BUILD_PATH) - for file in files_list: - os.remove(os.path.join(self.RPMBUILD_BUILD_PATH, file)) - except Exception as e: - logger.warning(e) - - def create_connection(self): - """ - 与数据库建立连接 - """ - logger.info("getting connection with source_clean.db ...") - try: - con = sqlite3.connect(self.database_path) - return con - except Error: - logger.error(Error) + files_list = os.listdir(self.rpmbuild_source_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_source_path, file)) + files_list = os.listdir(self.rpmbuild_build_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_build_path, file)) @staticmethod def get_sha256sum_from_url(con, url): @@ -127,6 +114,18 @@ class CheckSourceConsistency(BaseCheck): return row[0] return "" + def create_connection(self): + """ + 与数据库建立连接 + """ + logger.info("getting connection with source_clean.db ...") + try: + con = sqlite3.connect(self.database_path) + return con + except Error: + logger.error(Error) + return True + @staticmethod def get_sha256sum(package): """ @@ -185,7 +184,7 @@ class CheckSourceConsistency(BaseCheck): """ logger.info("reading spec file : %s ...", os.path.basename(spec_file)) - new_spec_file = os.path.join(self.RPMBUILD_SOURCE_PATH, "get_source.spec") + new_spec_file = os.path.join(self.rpmbuild_source_path, "get_source.spec") cond_source = re.compile("^Source0*") source_url = "" new_spec_content = "" @@ -201,9 +200,10 @@ class CheckSourceConsistency(BaseCheck): new_spec_content += line + os.linesep new_spec_content += self.get_prep_function(source_url) logger.info("generating new spec file ...") - f = open(new_spec_file, "w") - f.write(new_spec_content) - f.close() + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + modes = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(new_spec_file, flags, modes), 'w') as f: + f.write(new_spec_content) def get_prep_function(self, url): """ @@ -212,8 +212,8 @@ class CheckSourceConsistency(BaseCheck): logger.info("generating %prep function") function_content = "%prep" + os.linesep function_content += "source={0}".format(url) + os.linesep - function_content += "cd {0}".format(self.RPMBUILD_SOURCE_PATH) + os.linesep - function_content += "echo $source > {0}".format(self.TEMP_TXT_PATH) + os.linesep + function_content += "cd {0}".format(self.rpmbuild_source_path) + os.linesep + function_content += "echo $source > {0}".format(self.temp_txt_ppath) + os.linesep return function_content def do_rpmbuild(self): @@ -221,13 +221,13 @@ class CheckSourceConsistency(BaseCheck): 对新生成的spec文件执行rpmbuild """ logger.info("start to do rpmbuild") - new_spec_file = os.path.join(self.RPMBUILD_SOURCE_PATH, "get_source.spec") + new_spec_file = os.path.join(self.rpmbuild_source_path, "get_source.spec") res = os.system("rpmbuild -bp --nodeps {0} --define \"_topdir {1}\"".format(new_spec_file, self.rpmbuild_dir)) if res != 0: logger.error("do rpmbuild fail") - if not os.path.exists(self.TEMP_TXT_PATH): + if not os.path.exists(self.temp_txt_ppath): return "" - f = open(self.TEMP_TXT_PATH, "r") + f = open(self.temp_txt_ppath, "r") source_url = f.read().strip() f.close() return source_url -- Gitee From 822d1d0dc48a31918ce1296fa08e750090316c7a Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Tue, 20 Sep 2022 10:53:33 +0800 Subject: [PATCH 3/4] "adjust def order" --- .../source_consistency/check_consistency.py | 134 +++++++++--------- 1 file changed, 67 insertions(+), 67 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index b8aee55..b634e23 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -88,73 +88,6 @@ class CheckSourceConsistency(BaseCheck): os.mkdir(os.path.join(self.rpmbuild_dir, "SPECS")) os.mkdir(os.path.join(self.rpmbuild_dir, "SRPMS")) - def clear_temp(self): - """ - 清理生成的中间文件 - """ - if os.path.exists(self._work_dir): - shutil.rmtree(self._work_dir) - files_list = os.listdir(self.rpmbuild_source_path) - for file in files_list: - os.remove(os.path.join(self.rpmbuild_source_path, file)) - files_list = os.listdir(self.rpmbuild_build_path) - for file in files_list: - os.remove(os.path.join(self.rpmbuild_build_path, file)) - - @staticmethod - def get_sha256sum_from_url(con, url): - """ - 查询数据库，获取url的sha256sum值 - """ - logger.info("getting sha256sum of remote source package from source_clean.db ...") - cursor_obj = con.cursor() - cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) - rows = cursor_obj.fetchall() - for row in rows: - return row[0] - return "" - - def create_connection(self): - """ - 与数据库建立连接 - """ - logger.info("getting connection with source_clean.db ...") - try: - con = sqlite3.connect(self.database_path) - return con - except Error: - logger.error(Error) - return True - - @staticmethod - def get_sha256sum(package): - """ - 计算文件的sha256sum值 - """ - logger.info("getting sha256sum of native source package...") - native_sha256sum = "" - try: - with open(package, "rb") as f: - sha256obj = hashlib.sha256() - sha256obj.update(f.read()) - native_sha256sum = sha256obj.hexdigest() - except Exception as e: - logger.warning(e) - if native_sha256sum == "": - try: - native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] - except Exception as e: - logger.warning(e) - return native_sha256sum.strip() - - @staticmethod - def get_package_from_source(url): - """ - 从url中获取包名 - """ - package_name = url.split("/")[-1].strip() - return package_name - def get_source_url(self): """ 获取spec文件中的Source URL @@ -231,3 +164,70 @@ class CheckSourceConsistency(BaseCheck): source_url = f.read().strip() f.close() return source_url + + @staticmethod + def get_package_from_source(url): + """ + 从url中获取包名 + """ + package_name = url.split("/")[-1].strip() + return package_name + + @staticmethod + def get_sha256sum(package): + """ + 计算文件的sha256sum值 + """ + logger.info("getting sha256sum of native source package...") + native_sha256sum = "" + try: + with open(package, "rb") as f: + sha256obj = hashlib.sha256() + sha256obj.update(f.read()) + native_sha256sum = sha256obj.hexdigest() + except Exception as e: + logger.warning(e) + if native_sha256sum == "": + try: + native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] + except Exception as e: + logger.warning(e) + return native_sha256sum.strip() + + def create_connection(self): + """ + 与数据库建立连接 + """ + logger.info("getting connection with source_clean.db ...") + try: + con = sqlite3.connect(self.database_path) + return con + except Error: + logger.error(Error) + return True + + @staticmethod + def get_sha256sum_from_url(con, url): + """ + 查询数据库，获取url的sha256sum值 + """ + logger.info("getting sha256sum of remote source package from source_clean.db ...") + cursor_obj = con.cursor() + cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) + rows = cursor_obj.fetchall() + for row in rows: + return row[0] + return "" + + def clear_temp(self): + """ + 清理生成的中间文件 + """ + if os.path.exists(self._work_dir): + shutil.rmtree(self._work_dir) + files_list = os.listdir(self.rpmbuild_source_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_source_path, file)) + files_list = os.listdir(self.rpmbuild_build_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_build_path, file)) -- Gitee From 6cf4411ffcf165fd369ebcdc1ceb6edb0d2e2cb8 Mon Sep 17 00:00:00 2001 From: liangchengh <1242747392@qq.com> Date: Tue, 20 Sep 2022 11:12:37 +0800 Subject: [PATCH 4/4] "def order" --- .../source_consistency/check_consistency.py | 84 +++++++++---------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index b634e23..511808f 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -45,6 +45,48 @@ class CheckSourceConsistency(BaseCheck): finally: self.clear_temp() + @staticmethod + def get_package_from_source(url): + """ + 从url中获取包名 + """ + package_name = url.split("/")[-1].strip() + return package_name + + @staticmethod + def get_sha256sum(package): + """ + 计算文件的sha256sum值 + """ + logger.info("getting sha256sum of native source package...") + native_sha256sum = "" + try: + with open(package, "rb") as f: + sha256obj = hashlib.sha256() + sha256obj.update(f.read()) + native_sha256sum = sha256obj.hexdigest() + except Exception as e: + logger.warning(e) + if native_sha256sum == "": + try: + native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] + except Exception as e: + logger.warning(e) + return native_sha256sum.strip() + + @staticmethod + def get_sha256sum_from_url(con, url): + """ + 查询数据库，获取url的sha256sum值 + """ + logger.info("getting sha256sum of remote source package from source_clean.db ...") + cursor_obj = con.cursor() + cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) + rows = cursor_obj.fetchall() + for row in rows: + return row[0] + return "" + def check_source_consistency(self): """ 检查源码包是否一致 @@ -165,35 +207,6 @@ class CheckSourceConsistency(BaseCheck): f.close() return source_url - @staticmethod - def get_package_from_source(url): - """ - 从url中获取包名 - """ - package_name = url.split("/")[-1].strip() - return package_name - - @staticmethod - def get_sha256sum(package): - """ - 计算文件的sha256sum值 - """ - logger.info("getting sha256sum of native source package...") - native_sha256sum = "" - try: - with open(package, "rb") as f: - sha256obj = hashlib.sha256() - sha256obj.update(f.read()) - native_sha256sum = sha256obj.hexdigest() - except Exception as e: - logger.warning(e) - if native_sha256sum == "": - try: - native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] - except Exception as e: - logger.warning(e) - return native_sha256sum.strip() - def create_connection(self): """ 与数据库建立连接 @@ -206,19 +219,6 @@ class CheckSourceConsistency(BaseCheck): logger.error(Error) return True - @staticmethod - def get_sha256sum_from_url(con, url): - """ - 查询数据库，获取url的sha256sum值 - """ - logger.info("getting sha256sum of remote source package from source_clean.db ...") - cursor_obj = con.cursor() - cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) - rows = cursor_obj.fetchall() - for row in rows: - return row[0] - return "" - def clear_temp(self): """ 清理生成的中间文件 -- Gitee