diff --git a/src/ac/acl/source_consistency/__init__.py b/src/ac/acl/source_consistency/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py new file mode 100644 index 0000000000000000000000000000000000000000..511808feacd1f82719d98315b8374d1461f5c656 --- /dev/null +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -0,0 +1,233 @@ +import hashlib +import logging +import os +import re +import stat +import shutil +import sqlite3 +from sqlite3 import Error + +from src.ac.common.gitee_repo import GiteeRepo +from src.ac.framework.ac_base import BaseCheck +from src.ac.framework.ac_result import FAILED, SUCCESS, WARNING + +logger = logging.getLogger("ac") + + +class CheckSourceConsistency(BaseCheck): + """ + check source consistence + """ + + def __init__(self, workspace, repo, conf=None): + super(CheckSourceConsistency, self).__init__(workspace, repo, conf) + + self._work_dir = os.path.join(workspace, "source_consistency") + shutil.copytree(os.path.join(workspace, repo), self._work_dir) + self._repo = repo + self.temp_txt_ppath = os.path.join(self._work_dir, "temp.txt") + self.rpmbuild_dir = os.path.join(workspace, "rpmbuild") + self.rpmbuild_build_path = os.path.join(self.rpmbuild_dir, "BUILD") + self.rpmbuild_source_path = os.path.join(self.rpmbuild_dir, "SOURCE") + self.database_path = "source_clean.db" + + def __call__(self, *args, **kwargs): + """ + 入口函数 + :param args: + :param kwargs: + :return: + """ + logger.info("check %s source consistency ...", self._repo) + _ = not os.path.exists("log") and os.mkdir("log") + try: + return self.start_check_with_order("source_consistency") + finally: + self.clear_temp() + + @staticmethod + def get_package_from_source(url): + """ + 从url中获取包名 + """ + package_name = url.split("/")[-1].strip() + return package_name + + @staticmethod + def get_sha256sum(package): + """ + 计算文件的sha256sum值 + """ + logger.info("getting sha256sum of native source package...") + native_sha256sum = "" + try: + with open(package, "rb") as f: + sha256obj = hashlib.sha256() + sha256obj.update(f.read()) + native_sha256sum = sha256obj.hexdigest() + except Exception as e: + logger.warning(e) + if native_sha256sum == "": + try: + native_sha256sum = os.popen("sha256sum {0}".format(package)).read().split()[0] + except Exception as e: + logger.warning(e) + return native_sha256sum.strip() + + @staticmethod + def get_sha256sum_from_url(con, url): + """ + 查询数据库,获取url的sha256sum值 + """ + logger.info("getting sha256sum of remote source package from source_clean.db ...") + cursor_obj = con.cursor() + cursor_obj.execute("SELECT sha256sum FROM source_package WHERE url = ?", (url,)) + rows = cursor_obj.fetchall() + for row in rows: + return row[0] + return "" + + def check_source_consistency(self): + """ + 检查源码包是否一致 + """ + self.create_rpmbuild_dir() + source_url = self.get_source_url() + if source_url == "": + logger.warning("no valid source url") + return SUCCESS + + package_name = self.get_package_from_source(source_url) + if package_name not in os.listdir(self._work_dir): + logger.warning("no source package file") + return SUCCESS + + native_sha256sum = self.get_sha256sum(os.path.join(self._work_dir, package_name)) + if native_sha256sum == "": + logger.warning("get sha256sum of native source package failed") + return SUCCESS + + con = self.create_connection() + remote_sha256sum = self.get_sha256sum_from_url(con, source_url) + if remote_sha256sum == "": + logger.warning("no url in source_clean.db") + return SUCCESS + if native_sha256sum != remote_sha256sum: + logger.error("repo is inconsistency") + return FAILED + + return SUCCESS + + def create_rpmbuild_dir(self): + """ + 创建临时rpmbuild目录 + """ + os.mkdir(self.rpmbuild_dir) + os.mkdir(os.path.join(self.rpmbuild_dir, "BUILD")) + os.mkdir(os.path.join(self.rpmbuild_dir, "BUILDROOT")) + os.mkdir(os.path.join(self.rpmbuild_dir, "RPMS")) + os.mkdir(os.path.join(self.rpmbuild_dir, "SOURCE")) + os.mkdir(os.path.join(self.rpmbuild_dir, "SPECS")) + os.mkdir(os.path.join(self.rpmbuild_dir, "SRPMS")) + + def get_source_url(self): + """ + 获取spec文件中的Source URL + """ + files_list = os.listdir(self._work_dir) + if len(files_list) == 0: + logger.error("compressed fail") + return "" + if self._repo + ".spec" not in files_list: + logger.error("no spec file") + return "" + source_url = self.get_source_from_rpmbuild() + return source_url + + def get_source_from_rpmbuild(self): + """ + rpmbuild解析出可查询的Source URL + """ + spec_file = os.path.join(self._work_dir, self._repo + ".spec") + self.spec_read(spec_file) + source_url = self.do_rpmbuild() + return source_url + + def spec_read(self, spec_file): + """ + 读取spec文件并生成新的spec文件 + """ + logger.info("reading spec file : %s ...", os.path.basename(spec_file)) + + new_spec_file = os.path.join(self.rpmbuild_source_path, "get_source.spec") + cond_source = re.compile("^Source0*") + source_url = "" + new_spec_content = "" + for line in open(spec_file): + line = line.strip() + if line.startswith("%prep"): + break + elif cond_source.match(line) or re.match("^Source.*", line): + if source_url == "": + if ":" in line: + line_list = line.split(":") + source_url = ":".join(line_list[1:]).strip() + new_spec_content += line + os.linesep + new_spec_content += self.get_prep_function(source_url) + logger.info("generating new spec file ...") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + modes = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(new_spec_file, flags, modes), 'w') as f: + f.write(new_spec_content) + + def get_prep_function(self, url): + """ + 生成spec文件%prep部分的内容 + """ + logger.info("generating %prep function") + function_content = "%prep" + os.linesep + function_content += "source={0}".format(url) + os.linesep + function_content += "cd {0}".format(self.rpmbuild_source_path) + os.linesep + function_content += "echo $source > {0}".format(self.temp_txt_ppath) + os.linesep + return function_content + + def do_rpmbuild(self): + """ + 对新生成的spec文件执行rpmbuild + """ + logger.info("start to do rpmbuild") + new_spec_file = os.path.join(self.rpmbuild_source_path, "get_source.spec") + res = os.system("rpmbuild -bp --nodeps {0} --define \"_topdir {1}\"".format(new_spec_file, self.rpmbuild_dir)) + if res != 0: + logger.error("do rpmbuild fail") + if not os.path.exists(self.temp_txt_ppath): + return "" + f = open(self.temp_txt_ppath, "r") + source_url = f.read().strip() + f.close() + return source_url + + def create_connection(self): + """ + 与数据库建立连接 + """ + logger.info("getting connection with source_clean.db ...") + try: + con = sqlite3.connect(self.database_path) + return con + except Error: + logger.error(Error) + return True + + def clear_temp(self): + """ + 清理生成的中间文件 + """ + if os.path.exists(self._work_dir): + shutil.rmtree(self._work_dir) + files_list = os.listdir(self.rpmbuild_source_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_source_path, file)) + files_list = os.listdir(self.rpmbuild_build_path) + for file in files_list: + os.remove(os.path.join(self.rpmbuild_build_path, file)) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 4d7bc70ccc11d4fd22be64b47dab7179f3ac1524..f96be37280df0055eefd02ce90162421b0bb75b6 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -29,6 +29,10 @@ src-openeuler: exclude: True commit_msg: exclude: True + source_consistency: + hint: check_consistency + module: source_consistency.check_consistency + entry: CheckSourceConsistency openeuler: spec: exclude: True @@ -46,7 +50,7 @@ openeuler: hint: check_sca module: sca.check_sca entry: CheckSCA - deny_list: ["bishengjdk-17", "bishengjdk-8", "bishengjdk-11", "bishengjdk-riscv", "gcc", "kernel"] + deny_list: [] openlibing: hint: code module: openlibing.check_code @@ -58,10 +62,11 @@ openeuler: "isula-build", "cve-manager", "attest-tools", "oec-hardware", "itrustee_sdk", "wisdom-advisor", "isula-transform", " itrustee_client", "A-Tune-Collector", "itrustee_tzdriver", "website-v2", "yocto-poky", "bishengjdk-17", "bishengjdk-8", "bishengjdk-11", "bishengjdk-riscv", "powerapi", - "eagle", "dcs", "astream", "QARobot", "oec-application", "gala-gopher", "gala-anteater", - "gala-spider", "gala-ragdoll"] + "eagle", "dcs", "astream", "QARobot", "oec-application"] commit_msg: hint: commit_msg module: commit_msg.check_commit_msg entry: CheckCommitMsg allow_list: ["yocto-meta-openeuler"] + source_consistency: + exclude: True diff --git a/src/lib/trigger.sh b/src/lib/trigger.sh index 6a8585fd97c3c88cacb3b9f77999da163266deed..ccbc8ebb513727524d2a5dc9e1cd9e398a1fd8c9 100644 --- a/src/lib/trigger.sh +++ b/src/lib/trigger.sh @@ -26,6 +26,8 @@ function clearn_env() { EOF ) ssh -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR root@${repo_server} "$remote_dir_reset_cmd" + log_info "***** Start to copy db file *****" + scp -r -i ${SaveBuildRPM2Repo} -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@${repo_server}:/repo/soe/sql/source_clean.db . || log_info "file source_clean not exist" } # 开始下载kernel代码