From db2903b5c8278ef67f0f149261fd071de7e749d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Fri, 23 Dec 2022 07:44:53 +0000 Subject: [PATCH 01/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- .../source_consistency/check_consistency.py | 88 ++++--------------- 1 file changed, 17 insertions(+), 71 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 0e5cc92..7cbcbb4 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -1,13 +1,11 @@ import hashlib import logging import os -import re -import stat import shutil import sqlite3 +import subprocess from sqlite3 import Error -from src.ac.common.gitee_repo import GiteeRepo from src.ac.framework.ac_base import BaseCheck from src.ac.framework.ac_result import FAILED, SUCCESS, WARNING @@ -26,9 +24,6 @@ class CheckSourceConsistency(BaseCheck): shutil.copytree(os.path.join(workspace, repo), self._work_dir) self._repo = repo self.temp_txt_path = os.path.join(self._work_dir, "temp.txt") - self.rpmbuild_dir = os.path.join(workspace, "rpmbuild") - self.rpmbuild_build_path = os.path.join(self.rpmbuild_dir, "BUILD") - self.rpmbuild_sources_path = os.path.join(self.rpmbuild_dir, "SOURCES") self.database_path = "source_clean.db" self.con = self.create_connection() @@ -48,12 +43,15 @@ class CheckSourceConsistency(BaseCheck): self.con.close() self.clear_temp() - @staticmethod - def get_package_from_source(url): + def get_package_name(self, url): """ - 从url中获取包名 + 从文件列表或者url中获取包名 """ - package_name = url.split("/")[-1].strip() + ret = subprocess.Popen("ls -S {0} | grep -v .spec | grep -v .yaml | grep -v .patch | grep -v .md | head -n 1" + .format(self._work_dir), shell=True, stdout=subprocess.PIPE) + package_name = ret.stdout.read().decode('utf-8').strip() + if package_name == "": + package_name = os.path.basename(url) return package_name @staticmethod @@ -81,13 +79,12 @@ class CheckSourceConsistency(BaseCheck): """ 检查源码包是否一致 """ - os.makedirs(os.path.join(self.rpmbuild_dir, "SOURCES"), exist_ok=True) source_url = self.get_source_url() if source_url == "": logger.warning("no valid source url") return WARNING - package_name = self.get_package_from_source(source_url) + package_name = self.get_package_name(source_url) if package_name not in os.listdir(self._work_dir): logger.warning("no source package file") return WARNING @@ -125,71 +122,21 @@ class CheckSourceConsistency(BaseCheck): if spec_name == "": logger.error("no spec file, please check!") return "" - source_url = self.get_source_from_rpmbuild(spec_name) + source_url = self.get_source_from_spec(spec_name) return source_url - def get_source_from_rpmbuild(self, spec_name=""): + def get_source_from_spec(self, spec_name=""): """ - rpmbuild解析出可查询的Source URL + spec文件中得到可查询的Source URL """ if spec_name == "": spec_name = self._repo + ".spec" spec_file = os.path.join(self._work_dir, spec_name) - self.generate_new_spec(spec_file) - source_url = self.do_rpmbuild() - return source_url - - def generate_new_spec(self, spec_file): - """ - 读取spec文件并生成新的spec文件 - """ - logger.info("reading spec file : %s ...", os.path.basename(spec_file)) - - new_spec_file = os.path.join(self.rpmbuild_sources_path, "get_source.spec") - cond_source = re.compile("^Source0*") - source_url = "" - new_spec_content = "" - with open(spec_file) as f: - for line in f: - line = line.strip() - if line.startswith("%prep"): - break - elif cond_source.match(line) or re.match("^Source.*", line): - if source_url == "": - if ":" in line: - source_url = ":".join(line.split(":")[1:]).strip() - new_spec_content += line + os.linesep - new_spec_content += self.get_prep_function(source_url) - logger.info("generating new spec file ...") - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - modes = stat.S_IWUSR | stat.S_IRUSR - with os.fdopen(os.open(new_spec_file, flags, modes), 'w') as f: - f.write(new_spec_content) - - def get_prep_function(self, url): - """ - 生成spec文件%prep部分的内容 - """ - logger.info("generating %prep function") - function_content = "%prep" + os.linesep - function_content += "source={0}".format(url) + os.linesep - function_content += "cd {0}".format(self.rpmbuild_sources_path) + os.linesep - function_content += "echo $source > {0}".format(self.temp_txt_path) + os.linesep - return function_content - - def do_rpmbuild(self): - """ - 对新生成的spec文件执行rpmbuild - """ - logger.info("start to do rpmbuild") - new_spec_file = os.path.join(self.rpmbuild_sources_path, "get_source.spec") - res = os.system("rpmbuild -bp --nodeps {0} --define \"_topdir {1}\"".format(new_spec_file, self.rpmbuild_dir)) - if res != 0: - logger.error("do rpmbuild fail") - if not os.path.exists(self.temp_txt_path): - return "" - with open(self.temp_txt_path, "r") as f: - source_url = f.read().strip() + ret = subprocess.Popen("spectool -S {0}".format(spec_file), shell=True, stdout=subprocess.PIPE) + content = ret.stdout.read().decode('utf-8').strip() + source_url = content.split(os.linesep)[0].strip() if os.linesep in content else content.strip() + if ":" in source_url: + source_url = ":".join(source_url.split(":")[1:]).strip() return source_url def create_connection(self): @@ -226,4 +173,3 @@ class CheckSourceConsistency(BaseCheck): """ if os.path.exists(self._work_dir): shutil.rmtree(self._work_dir) - shutil.rmtree(self.rpmbuild_dir) -- Gitee From 2872b3dd1be9c75de8cc8dbf5db552be6c2c83bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Tue, 27 Dec 2022 07:06:28 +0000 Subject: [PATCH 02/24] update src/ac/framework/ac.yaml. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/framework/ac.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 991cb07..7820bb4 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -30,7 +30,9 @@ src-openeuler: commit_msg: exclude: True source_consistency: - exclude: True + hint: check_consistency + module: source_consistency.check_consistency + entry: CheckSourceConsistency openeuler: spec: exclude: True -- Gitee From 4ec7dbe1ac565f847d98e5879a3970fd7493eaa3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Tue, 27 Dec 2022 07:18:55 +0000 Subject: [PATCH 03/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 7cbcbb4..17351e4 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -132,6 +132,9 @@ class CheckSourceConsistency(BaseCheck): if spec_name == "": spec_name = self._repo + ".spec" spec_file = os.path.join(self._work_dir, spec_name) + ret0 = os.system("yum install -y spectool") + if ret0 != 0: + print("-------->>>>") ret = subprocess.Popen("spectool -S {0}".format(spec_file), shell=True, stdout=subprocess.PIPE) content = ret.stdout.read().decode('utf-8').strip() source_url = content.split(os.linesep)[0].strip() if os.linesep in content else content.strip() -- Gitee From 4dc5c07a5b43aff58d2fbbab73b5a3f90ff28d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Tue, 27 Dec 2022 07:22:24 +0000 Subject: [PATCH 04/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 17351e4..df24bba 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -132,7 +132,7 @@ class CheckSourceConsistency(BaseCheck): if spec_name == "": spec_name = self._repo + ".spec" spec_file = os.path.join(self._work_dir, spec_name) - ret0 = os.system("yum install -y spectool") + ret0 = os.system("sudo yum install -y spectool") if ret0 != 0: print("-------->>>>") ret = subprocess.Popen("spectool -S {0}".format(spec_file), shell=True, stdout=subprocess.PIPE) -- Gitee From 8fb20935f8fb5ef19a5a04a00591dba2961fb9fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Tue, 27 Dec 2022 11:18:20 +0000 Subject: [PATCH 05/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- .../source_consistency/check_consistency.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index df24bba..02e2eb7 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -83,6 +83,9 @@ class CheckSourceConsistency(BaseCheck): if source_url == "": logger.warning("no valid source url") return WARNING + elif source_url == "IndexError": + logger.warning("no effective source value") + return WARNING package_name = self.get_package_name(source_url) if package_name not in os.listdir(self._work_dir): @@ -132,14 +135,31 @@ class CheckSourceConsistency(BaseCheck): if spec_name == "": spec_name = self._repo + ".spec" spec_file = os.path.join(self._work_dir, spec_name) - ret0 = os.system("sudo yum install -y spectool") - if ret0 != 0: - print("-------->>>>") + if not os.path.exists(spec_file): + temp_file_list = os.listdir(self._work_dir) + spec_file_list = [] + for temp_file in temp_file_list: + if temp_file.endswith(".spec"): + spec_file_list.append(temp_file) + if len(spec_file_list) == 1: + spec_file = os.getcwd() + os.path.sep + "gitee_code" + os.path.sep + self._repo + os.path.sep + \ + spec_file_list[0] + elif len(spec_file_list) > 1: + for s_file in spec_file_list: + if self._repo in s_file or s_file in self._repo: + spec_file = os.getcwd() + os.path.sep + "gitee_code" + os.path.sep + self._repo + os.path.sep +\ + s_file + else: + return "" ret = subprocess.Popen("spectool -S {0}".format(spec_file), shell=True, stdout=subprocess.PIPE) content = ret.stdout.read().decode('utf-8').strip() source_url = content.split(os.linesep)[0].strip() if os.linesep in content else content.strip() if ":" in source_url: source_url = ":".join(source_url.split(":")[1:]).strip() + elif "No such file or directory" in source_url: + return "" + else: + return "IndexError" return source_url def create_connection(self): -- Gitee From c1fe7d246cfae918db92e4e463f5dc1369b1e7f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Tue, 27 Dec 2022 11:23:11 +0000 Subject: [PATCH 06/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 02e2eb7..c19a62a 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -99,6 +99,7 @@ class CheckSourceConsistency(BaseCheck): remote_sha256sum = self.get_sha256sum_from_url(source_url) if remote_sha256sum == "": + print("native_sha256sum======>>>", native_sha256sum) logger.warning("no url in source_clean.db") return WARNING if native_sha256sum != remote_sha256sum: @@ -160,6 +161,7 @@ class CheckSourceConsistency(BaseCheck): return "" else: return "IndexError" + print("source_url=====>>>>>", source_url) return source_url def create_connection(self): -- Gitee From ab384e3b4bb441edbe701e8406dbf616589dd39c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 02:19:13 +0000 Subject: [PATCH 07/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- .../source_consistency/check_consistency.py | 80 +++++++++++++++++-- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index c19a62a..9912e76 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -1,6 +1,8 @@ import hashlib import logging import os +import re +import stat import shutil import sqlite3 import subprocess @@ -24,6 +26,9 @@ class CheckSourceConsistency(BaseCheck): shutil.copytree(os.path.join(workspace, repo), self._work_dir) self._repo = repo self.temp_txt_path = os.path.join(self._work_dir, "temp.txt") + self.rpmbuild_dir = os.path.join(workspace, "rpmbuild") + self.rpmbuild_build_path = os.path.join(self.rpmbuild_dir, "BUILD") + self.rpmbuild_sources_path = os.path.join(self.rpmbuild_dir, "SOURCES") self.database_path = "source_clean.db" self.con = self.create_connection() @@ -79,13 +84,11 @@ class CheckSourceConsistency(BaseCheck): """ 检查源码包是否一致 """ + os.makedirs(os.path.join(self.rpmbuild_dir, "SOURCES"), exist_ok=True) source_url = self.get_source_url() if source_url == "": logger.warning("no valid source url") return WARNING - elif source_url == "IndexError": - logger.warning("no effective source value") - return WARNING package_name = self.get_package_name(source_url) if package_name not in os.listdir(self._work_dir): @@ -99,7 +102,6 @@ class CheckSourceConsistency(BaseCheck): remote_sha256sum = self.get_sha256sum_from_url(source_url) if remote_sha256sum == "": - print("native_sha256sum======>>>", native_sha256sum) logger.warning("no url in source_clean.db") return WARNING if native_sha256sum != remote_sha256sum: @@ -127,8 +129,21 @@ class CheckSourceConsistency(BaseCheck): logger.error("no spec file, please check!") return "" source_url = self.get_source_from_spec(spec_name) + if source_url == "": + source_url = self.get_source_from_rpmbuild(spec_name) return source_url + def get_source_from_rpmbuild(self, spec_name=""): + """ + rpmbuild解析出可查询的Source URL + """ + if spec_name == "": + spec_name = self._repo + ".spec" + spec_file = os.path.join(self._work_dir, spec_name) + self.generate_new_spec(spec_file) + source_url = self.do_rpmbuild() + return source_url + def get_source_from_spec(self, spec_name=""): """ spec文件中得到可查询的Source URL @@ -159,9 +174,59 @@ class CheckSourceConsistency(BaseCheck): source_url = ":".join(source_url.split(":")[1:]).strip() elif "No such file or directory" in source_url: return "" - else: - return "IndexError" - print("source_url=====>>>>>", source_url) + return source_url + + def generate_new_spec(self, spec_file): + """ + 读取spec文件并生成新的spec文件 + """ + logger.info("reading spec file : %s ...", os.path.basename(spec_file)) + + new_spec_file = os.path.join(self.rpmbuild_sources_path, "get_source.spec") + cond_source = re.compile("^Source0*") + source_url = "" + new_spec_content = "" + with open(spec_file) as f: + for line in f: + line = line.strip() + if line.startswith("%prep"): + break + elif cond_source.match(line) or re.match("^Source.*", line): + if source_url == "": + if ":" in line: + source_url = ":".join(line.split(":")[1:]).strip() + new_spec_content += line + os.linesep + new_spec_content += self.get_prep_function(source_url) + logger.info("generating new spec file ...") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + modes = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(new_spec_file, flags, modes), 'w') as f: + f.write(new_spec_content) + + def get_prep_function(self, url): + """ + 生成spec文件%prep部分的内容 + """ + logger.info("generating %prep function") + function_content = "%prep" + os.linesep + function_content += "source={0}".format(url) + os.linesep + function_content += "cd {0}".format(self.rpmbuild_sources_path) + os.linesep + function_content += "echo $source > {0}".format(self.temp_txt_path) + os.linesep + return function_content + + def do_rpmbuild(self): + """ + 对新生成的spec文件执行rpmbuild + """ + logger.info("start to do rpmbuild") + new_spec_file = os.path.join(self.rpmbuild_sources_path, "get_source.spec") + res = os.system("rpmbuild -bp --nodeps {0} --define \"_topdir {1}\"".format(new_spec_file, self.rpmbuild_dir)) + if res != 0: + logger.error("do rpmbuild fail") + if not os.path.exists(self.temp_txt_path): + return "" + with open(self.temp_txt_path, "r") as f: + source_url = f.read().strip() return source_url def create_connection(self): @@ -198,3 +263,4 @@ class CheckSourceConsistency(BaseCheck): """ if os.path.exists(self._work_dir): shutil.rmtree(self._work_dir) + shutil.rmtree(self.rpmbuild_dir) -- Gitee From 69687e3213277f36e34b22513ad6e19e82b0a5ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 02:26:40 +0000 Subject: [PATCH 08/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 9912e76..698914e 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -101,6 +101,7 @@ class CheckSourceConsistency(BaseCheck): return WARNING remote_sha256sum = self.get_sha256sum_from_url(source_url) + remote_sha256sum = native_sha256sum if remote_sha256sum == "": logger.warning("no url in source_clean.db") return WARNING -- Gitee From 274112be578cc6ebcac7f6519ddd9161a8698280 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 02:31:02 +0000 Subject: [PATCH 09/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 698914e..529ca28 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -101,7 +101,7 @@ class CheckSourceConsistency(BaseCheck): return WARNING remote_sha256sum = self.get_sha256sum_from_url(source_url) - remote_sha256sum = native_sha256sum + remote_sha256sum = "132456654" if remote_sha256sum == "": logger.warning("no url in source_clean.db") return WARNING -- Gitee From 1e775756434a092d1fb2e06b73bf3ac2283590e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 02:41:01 +0000 Subject: [PATCH 10/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 529ca28..9912e76 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -101,7 +101,6 @@ class CheckSourceConsistency(BaseCheck): return WARNING remote_sha256sum = self.get_sha256sum_from_url(source_url) - remote_sha256sum = "132456654" if remote_sha256sum == "": logger.warning("no url in source_clean.db") return WARNING -- Gitee From 18bd5b2c97ac93a213a3cd04595b2c8c49501d4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 02:49:02 +0000 Subject: [PATCH 11/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 9912e76..1465d8c 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -57,6 +57,7 @@ class CheckSourceConsistency(BaseCheck): package_name = ret.stdout.read().decode('utf-8').strip() if package_name == "": package_name = os.path.basename(url) + logger.info("=========>>>" + package_name) return package_name @staticmethod -- Gitee From 84387a896c549944b5e62553d40205ab67f19a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 02:55:18 +0000 Subject: [PATCH 12/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 1465d8c..5f0a506 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -92,6 +92,7 @@ class CheckSourceConsistency(BaseCheck): return WARNING package_name = self.get_package_name(source_url) + logger.info("=======>>>", " ".join(os.listdir(self._work_dir))) if package_name not in os.listdir(self._work_dir): logger.warning("no source package file") return WARNING -- Gitee From a75b9709934a04a90ff0f86d127797e8182a7d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 02:57:41 +0000 Subject: [PATCH 13/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 5f0a506..d549b20 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -92,7 +92,7 @@ class CheckSourceConsistency(BaseCheck): return WARNING package_name = self.get_package_name(source_url) - logger.info("=======>>>", " ".join(os.listdir(self._work_dir))) + logger.info("=======>>>" + " ".join(os.listdir(self._work_dir))) if package_name not in os.listdir(self._work_dir): logger.warning("no source package file") return WARNING -- Gitee From 2dcbbb3e08c7cc1d24bcf59c0a5a7213d35bf926 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 03:08:50 +0000 Subject: [PATCH 14/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index d549b20..4162318 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -92,7 +92,7 @@ class CheckSourceConsistency(BaseCheck): return WARNING package_name = self.get_package_name(source_url) - logger.info("=======>>>" + " ".join(os.listdir(self._work_dir))) + os.system("rm -f {0}".format(os.path.join(self._work_dir, package_name))) if package_name not in os.listdir(self._work_dir): logger.warning("no source package file") return WARNING -- Gitee From e4d103325fa3d6721b0c34d8b39d38ba8843237a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 03:24:12 +0000 Subject: [PATCH 15/24] update src/ac/acl/source_consistency/check_consistency.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 --- src/ac/acl/source_consistency/check_consistency.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 4162318..590ac0a 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -57,7 +57,6 @@ class CheckSourceConsistency(BaseCheck): package_name = ret.stdout.read().decode('utf-8').strip() if package_name == "": package_name = os.path.basename(url) - logger.info("=========>>>" + package_name) return package_name @staticmethod @@ -92,7 +91,6 @@ class CheckSourceConsistency(BaseCheck): return WARNING package_name = self.get_package_name(source_url) - os.system("rm -f {0}".format(os.path.join(self._work_dir, package_name))) if package_name not in os.listdir(self._work_dir): logger.warning("no source package file") return WARNING @@ -131,6 +129,7 @@ class CheckSourceConsistency(BaseCheck): logger.error("no spec file, please check!") return "" source_url = self.get_source_from_spec(spec_name) + # If program can't get source url from spec, try to get source url by rpmbuild if source_url == "": source_url = self.get_source_from_rpmbuild(spec_name) return source_url @@ -145,7 +144,7 @@ class CheckSourceConsistency(BaseCheck): self.generate_new_spec(spec_file) source_url = self.do_rpmbuild() return source_url - + def get_source_from_spec(self, spec_name=""): """ spec文件中得到可查询的Source URL -- Gitee From 66af8bf8845a18e3d55ce20a3f632be42ffe17e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 11:55:38 +0800 Subject: [PATCH 16/24] pylint --- .../source_consistency/check_consistency.py | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 590ac0a..7c7233e 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -63,6 +63,8 @@ class CheckSourceConsistency(BaseCheck): def get_sha256sum(package): """ 计算文件的sha256sum值 + :param package:包路径 + :return: """ logger.info("getting sha256sum of native source package...") native_sha256sum = "" @@ -83,8 +85,10 @@ class CheckSourceConsistency(BaseCheck): def check_source_consistency(self): """ 检查源码包是否一致 + :return: """ - os.makedirs(os.path.join(self.rpmbuild_dir, "SOURCES"), exist_ok=True) + if not os.path.exists(os.path.join(self.rpmbuild_dir, "SOURCES")): + os.makedirs(os.path.join(self.rpmbuild_dir, "SOURCES")) source_url = self.get_source_url() if source_url == "": logger.warning("no valid source url") @@ -113,6 +117,7 @@ class CheckSourceConsistency(BaseCheck): def get_source_url(self): """ 获取spec文件中的Source URL + :return: """ spec_name = "" files_list = os.listdir(self._work_dir) @@ -137,6 +142,8 @@ class CheckSourceConsistency(BaseCheck): def get_source_from_rpmbuild(self, spec_name=""): """ rpmbuild解析出可查询的Source URL + :param spec_name:spec文件名 + :return: """ if spec_name == "": spec_name = self._repo + ".spec" @@ -148,6 +155,8 @@ class CheckSourceConsistency(BaseCheck): def get_source_from_spec(self, spec_name=""): """ spec文件中得到可查询的Source URL + :param spec_name:spec文件名 + :return: """ if spec_name == "": spec_name = self._repo + ".spec" @@ -180,6 +189,8 @@ class CheckSourceConsistency(BaseCheck): def generate_new_spec(self, spec_file): """ 读取spec文件并生成新的spec文件 + :param spec_file:spec文件名 + :return: """ logger.info("reading spec file : %s ...", os.path.basename(spec_file)) @@ -207,6 +218,8 @@ class CheckSourceConsistency(BaseCheck): def get_prep_function(self, url): """ 生成spec文件%prep部分的内容 + :param url:source0的值 + :return: """ logger.info("generating %prep function") function_content = "%prep" + os.linesep @@ -218,6 +231,7 @@ class CheckSourceConsistency(BaseCheck): def do_rpmbuild(self): """ 对新生成的spec文件执行rpmbuild + :return: """ logger.info("start to do rpmbuild") new_spec_file = os.path.join(self.rpmbuild_sources_path, "get_source.spec") @@ -233,6 +247,7 @@ class CheckSourceConsistency(BaseCheck): def create_connection(self): """ 与数据库建立连接 + :return: """ logger.info("getting connection with source_clean.db ...") try: @@ -246,6 +261,8 @@ class CheckSourceConsistency(BaseCheck): def get_sha256sum_from_url(self, url): """ 查询数据库，获取url的sha256sum值 + :param url:source0的值 + :return: """ logger.info("getting sha256sum of remote source package from source_clean.db ...") if self.con is None: @@ -261,6 +278,7 @@ class CheckSourceConsistency(BaseCheck): def clear_temp(self): """ 清理生成的中间文件 + :return: """ if os.path.exists(self._work_dir): shutil.rmtree(self._work_dir) -- Gitee From 241bb9498e296bb5a32dae54451c4432a3feb028 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 14:39:07 +0800 Subject: [PATCH 17/24] pylint --- .../source_consistency/check_consistency.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 7c7233e..7417984 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -48,17 +48,6 @@ class CheckSourceConsistency(BaseCheck): self.con.close() self.clear_temp() - def get_package_name(self, url): - """ - 从文件列表或者url中获取包名 - """ - ret = subprocess.Popen("ls -S {0} | grep -v .spec | grep -v .yaml | grep -v .patch | grep -v .md | head -n 1" - .format(self._work_dir), shell=True, stdout=subprocess.PIPE) - package_name = ret.stdout.read().decode('utf-8').strip() - if package_name == "": - package_name = os.path.basename(url) - return package_name - @staticmethod def get_sha256sum(package): """ @@ -82,6 +71,18 @@ class CheckSourceConsistency(BaseCheck): logger.warning(e) return native_sha256sum.strip() + def get_package_name(self, url): + """ + 从文件列表或者url中获取包名 + """ + ret = subprocess.check_output(["ls", "-S", self._work_dir, "|", "grep", "-v", ".spec", "|", "grep", "-v", + ".yaml", "|", "grep", "-v", ".patch", "|", "grep", "-v", ".md", "|", "head", + "-n", "1"], shell=False) + package_name = ret.decode().strip() + if package_name == "": + package_name = os.path.basename(url) + return package_name + def check_source_consistency(self): """ 检查源码包是否一致 @@ -177,7 +178,7 @@ class CheckSourceConsistency(BaseCheck): s_file else: return "" - ret = subprocess.Popen("spectool -S {0}".format(spec_file), shell=True, stdout=subprocess.PIPE) + ret = subprocess.check_output(["spectool", "-S", spec_file], shell=False) content = ret.stdout.read().decode('utf-8').strip() source_url = content.split(os.linesep)[0].strip() if os.linesep in content else content.strip() if ":" in source_url: -- Gitee From 981c8984f820ee97e53f4faf64846106cefaa678 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 14:50:09 +0800 Subject: [PATCH 18/24] pylint --- src/ac/acl/source_consistency/check_consistency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 7417984..6f63c61 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -75,7 +75,7 @@ class CheckSourceConsistency(BaseCheck): """ 从文件列表或者url中获取包名 """ - ret = subprocess.check_output(["ls", "-S", self._work_dir, "|", "grep", "-v", ".spec", "|", "grep", "-v", + ret = subprocess.check_output(["/usr/bin/ls", "-S", self._work_dir, "|", "grep", "-v", ".spec", "|", "grep", "-v", ".yaml", "|", "grep", "-v", ".patch", "|", "grep", "-v", ".md", "|", "head", "-n", "1"], shell=False) package_name = ret.decode().strip() @@ -178,7 +178,7 @@ class CheckSourceConsistency(BaseCheck): s_file else: return "" - ret = subprocess.check_output(["spectool", "-S", spec_file], shell=False) + ret = subprocess.check_output(["/usr/bin/spectool", "-S", spec_file], shell=False) content = ret.stdout.read().decode('utf-8').strip() source_url = content.split(os.linesep)[0].strip() if os.linesep in content else content.strip() if ":" in source_url: -- Gitee From 16526e306adb70830ba79715525e19558351644c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 14:55:23 +0800 Subject: [PATCH 19/24] pylint --- src/ac/acl/source_consistency/check_consistency.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 6f63c61..c71131c 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -75,9 +75,9 @@ class CheckSourceConsistency(BaseCheck): """ 从文件列表或者url中获取包名 """ - ret = subprocess.check_output(["/usr/bin/ls", "-S", self._work_dir, "|", "grep", "-v", ".spec", "|", "grep", "-v", - ".yaml", "|", "grep", "-v", ".patch", "|", "grep", "-v", ".md", "|", "head", - "-n", "1"], shell=False) + ret = subprocess.check_output(["/usr/bin/ls", "-S", self._work_dir, "|", "grep", "-v", ".spec", "|", "grep", + "-v", ".yaml", "|", "grep", "-v", ".patch", "|", "grep", "-v", ".md", "|", + "head", "-n", "1"], shell=False) package_name = ret.decode().strip() if package_name == "": package_name = os.path.basename(url) -- Gitee From 009a54fb09b7c1ab389f7b1c77e1c2e23ee46593 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 15:10:30 +0800 Subject: [PATCH 20/24] bug --- src/ac/acl/source_consistency/check_consistency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index c71131c..192f440 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -179,7 +179,7 @@ class CheckSourceConsistency(BaseCheck): else: return "" ret = subprocess.check_output(["/usr/bin/spectool", "-S", spec_file], shell=False) - content = ret.stdout.read().decode('utf-8').strip() + content = ret.decode('utf-8').strip() source_url = content.split(os.linesep)[0].strip() if os.linesep in content else content.strip() if ":" in source_url: source_url = ":".join(source_url.split(":")[1:]).strip() -- Gitee From a1c23cf2105c1a018c8ec546ebfe7ba50c57b35f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 15:19:16 +0800 Subject: [PATCH 21/24] bug --- src/ac/acl/source_consistency/check_consistency.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 192f440..e9965e8 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -77,7 +77,7 @@ class CheckSourceConsistency(BaseCheck): """ ret = subprocess.check_output(["/usr/bin/ls", "-S", self._work_dir, "|", "grep", "-v", ".spec", "|", "grep", "-v", ".yaml", "|", "grep", "-v", ".patch", "|", "grep", "-v", ".md", "|", - "head", "-n", "1"], shell=False) + "head", "-n", "1"], shell=True) package_name = ret.decode().strip() if package_name == "": package_name = os.path.basename(url) @@ -178,7 +178,7 @@ class CheckSourceConsistency(BaseCheck): s_file else: return "" - ret = subprocess.check_output(["/usr/bin/spectool", "-S", spec_file], shell=False) + ret = subprocess.check_output(["/usr/bin/spectool", "-S", spec_file], shell=True) content = ret.decode('utf-8').strip() source_url = content.split(os.linesep)[0].strip() if os.linesep in content else content.strip() if ":" in source_url: -- Gitee From 72046fb790ae9249d521a35159de0655b0c934b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 15:29:27 +0800 Subject: [PATCH 22/24] bug --- src/ac/acl/source_consistency/check_consistency.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index e9965e8..37945db 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -178,7 +178,7 @@ class CheckSourceConsistency(BaseCheck): s_file else: return "" - ret = subprocess.check_output(["/usr/bin/spectool", "-S", spec_file], shell=True) + ret = subprocess.check_output(["/usr/bin/spectool", "-S", spec_file], shell=False) content = ret.decode('utf-8').strip() source_url = content.split(os.linesep)[0].strip() if os.linesep in content else content.strip() if ":" in source_url: -- Gitee From 52c0cec2bc3aed07bb46d3df9e4a7921c8aaae64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Thu, 29 Dec 2022 16:14:51 +0800 Subject: [PATCH 23/24] bug --- src/ac/acl/source_consistency/check_consistency.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 37945db..fa5466e 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -75,10 +75,8 @@ class CheckSourceConsistency(BaseCheck): """ 从文件列表或者url中获取包名 """ - ret = subprocess.check_output(["/usr/bin/ls", "-S", self._work_dir, "|", "grep", "-v", ".spec", "|", "grep", - "-v", ".yaml", "|", "grep", "-v", ".patch", "|", "grep", "-v", ".md", "|", - "head", "-n", "1"], shell=True) - package_name = ret.decode().strip() + package_name = os.popen("ls -S {0} |grep -v .spec |grep -v .yaml |grep -v .patch |grep -v .md |head -n " + "1".format(self._work_dir)).read().split()[0] if package_name == "": package_name = os.path.basename(url) return package_name -- Gitee From b33e0fc4342c2e134cdada0d564946c1affe779d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=82=B1=E5=A0=82=E7=8F=82?= Date: Tue, 27 Dec 2022 07:06:28 +0000 Subject: [PATCH 24/24] update src/ac/framework/ac.yaml. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 update src/ac/acl/source_consistency/check_consistency.py. Signed-off-by: 邱堂珂 pylint pylint pylint pylint bug bug bug bug --- .../source_consistency/check_consistency.py | 135 ++++++++++++++++-- src/ac/framework/ac.yaml | 4 +- 2 files changed, 125 insertions(+), 14 deletions(-) diff --git a/src/ac/acl/source_consistency/check_consistency.py b/src/ac/acl/source_consistency/check_consistency.py index 7cbcbb4..fa5466e 100644 --- a/src/ac/acl/source_consistency/check_consistency.py +++ b/src/ac/acl/source_consistency/check_consistency.py @@ -1,6 +1,8 @@ import hashlib import logging import os +import re +import stat import shutil import sqlite3 import subprocess @@ -24,6 +26,9 @@ class CheckSourceConsistency(BaseCheck): shutil.copytree(os.path.join(workspace, repo), self._work_dir) self._repo = repo self.temp_txt_path = os.path.join(self._work_dir, "temp.txt") + self.rpmbuild_dir = os.path.join(workspace, "rpmbuild") + self.rpmbuild_build_path = os.path.join(self.rpmbuild_dir, "BUILD") + self.rpmbuild_sources_path = os.path.join(self.rpmbuild_dir, "SOURCES") self.database_path = "source_clean.db" self.con = self.create_connection() @@ -43,21 +48,12 @@ class CheckSourceConsistency(BaseCheck): self.con.close() self.clear_temp() - def get_package_name(self, url): - """ - 从文件列表或者url中获取包名 - """ - ret = subprocess.Popen("ls -S {0} | grep -v .spec | grep -v .yaml | grep -v .patch | grep -v .md | head -n 1" - .format(self._work_dir), shell=True, stdout=subprocess.PIPE) - package_name = ret.stdout.read().decode('utf-8').strip() - if package_name == "": - package_name = os.path.basename(url) - return package_name - @staticmethod def get_sha256sum(package): """ 计算文件的sha256sum值 + :param package:包路径 + :return: """ logger.info("getting sha256sum of native source package...") native_sha256sum = "" @@ -75,10 +71,23 @@ class CheckSourceConsistency(BaseCheck): logger.warning(e) return native_sha256sum.strip() + def get_package_name(self, url): + """ + 从文件列表或者url中获取包名 + """ + package_name = os.popen("ls -S {0} |grep -v .spec |grep -v .yaml |grep -v .patch |grep -v .md |head -n " + "1".format(self._work_dir)).read().split()[0] + if package_name == "": + package_name = os.path.basename(url) + return package_name + def check_source_consistency(self): """ 检查源码包是否一致 + :return: """ + if not os.path.exists(os.path.join(self.rpmbuild_dir, "SOURCES")): + os.makedirs(os.path.join(self.rpmbuild_dir, "SOURCES")) source_url = self.get_source_url() if source_url == "": logger.warning("no valid source url") @@ -107,6 +116,7 @@ class CheckSourceConsistency(BaseCheck): def get_source_url(self): """ 获取spec文件中的Source URL + :return: """ spec_name = "" files_list = os.listdir(self._work_dir) @@ -123,25 +133,120 @@ class CheckSourceConsistency(BaseCheck): logger.error("no spec file, please check!") return "" source_url = self.get_source_from_spec(spec_name) + # If program can't get source url from spec, try to get source url by rpmbuild + if source_url == "": + source_url = self.get_source_from_rpmbuild(spec_name) + return source_url + + def get_source_from_rpmbuild(self, spec_name=""): + """ + rpmbuild解析出可查询的Source URL + :param spec_name:spec文件名 + :return: + """ + if spec_name == "": + spec_name = self._repo + ".spec" + spec_file = os.path.join(self._work_dir, spec_name) + self.generate_new_spec(spec_file) + source_url = self.do_rpmbuild() return source_url def get_source_from_spec(self, spec_name=""): """ spec文件中得到可查询的Source URL + :param spec_name:spec文件名 + :return: """ if spec_name == "": spec_name = self._repo + ".spec" spec_file = os.path.join(self._work_dir, spec_name) - ret = subprocess.Popen("spectool -S {0}".format(spec_file), shell=True, stdout=subprocess.PIPE) - content = ret.stdout.read().decode('utf-8').strip() + if not os.path.exists(spec_file): + temp_file_list = os.listdir(self._work_dir) + spec_file_list = [] + for temp_file in temp_file_list: + if temp_file.endswith(".spec"): + spec_file_list.append(temp_file) + if len(spec_file_list) == 1: + spec_file = os.getcwd() + os.path.sep + "gitee_code" + os.path.sep + self._repo + os.path.sep + \ + spec_file_list[0] + elif len(spec_file_list) > 1: + for s_file in spec_file_list: + if self._repo in s_file or s_file in self._repo: + spec_file = os.getcwd() + os.path.sep + "gitee_code" + os.path.sep + self._repo + os.path.sep +\ + s_file + else: + return "" + ret = subprocess.check_output(["/usr/bin/spectool", "-S", spec_file], shell=False) + content = ret.decode('utf-8').strip() source_url = content.split(os.linesep)[0].strip() if os.linesep in content else content.strip() if ":" in source_url: source_url = ":".join(source_url.split(":")[1:]).strip() + elif "No such file or directory" in source_url: + return "" + return source_url + + def generate_new_spec(self, spec_file): + """ + 读取spec文件并生成新的spec文件 + :param spec_file:spec文件名 + :return: + """ + logger.info("reading spec file : %s ...", os.path.basename(spec_file)) + + new_spec_file = os.path.join(self.rpmbuild_sources_path, "get_source.spec") + cond_source = re.compile("^Source0*") + source_url = "" + new_spec_content = "" + with open(spec_file) as f: + for line in f: + line = line.strip() + if line.startswith("%prep"): + break + elif cond_source.match(line) or re.match("^Source.*", line): + if source_url == "": + if ":" in line: + source_url = ":".join(line.split(":")[1:]).strip() + new_spec_content += line + os.linesep + new_spec_content += self.get_prep_function(source_url) + logger.info("generating new spec file ...") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + modes = stat.S_IWUSR | stat.S_IRUSR + with os.fdopen(os.open(new_spec_file, flags, modes), 'w') as f: + f.write(new_spec_content) + + def get_prep_function(self, url): + """ + 生成spec文件%prep部分的内容 + :param url:source0的值 + :return: + """ + logger.info("generating %prep function") + function_content = "%prep" + os.linesep + function_content += "source={0}".format(url) + os.linesep + function_content += "cd {0}".format(self.rpmbuild_sources_path) + os.linesep + function_content += "echo $source > {0}".format(self.temp_txt_path) + os.linesep + return function_content + + def do_rpmbuild(self): + """ + 对新生成的spec文件执行rpmbuild + :return: + """ + logger.info("start to do rpmbuild") + new_spec_file = os.path.join(self.rpmbuild_sources_path, "get_source.spec") + res = os.system("rpmbuild -bp --nodeps {0} --define \"_topdir {1}\"".format(new_spec_file, self.rpmbuild_dir)) + if res != 0: + logger.error("do rpmbuild fail") + if not os.path.exists(self.temp_txt_path): + return "" + with open(self.temp_txt_path, "r") as f: + source_url = f.read().strip() return source_url def create_connection(self): """ 与数据库建立连接 + :return: """ logger.info("getting connection with source_clean.db ...") try: @@ -155,6 +260,8 @@ class CheckSourceConsistency(BaseCheck): def get_sha256sum_from_url(self, url): """ 查询数据库，获取url的sha256sum值 + :param url:source0的值 + :return: """ logger.info("getting sha256sum of remote source package from source_clean.db ...") if self.con is None: @@ -170,6 +277,8 @@ class CheckSourceConsistency(BaseCheck): def clear_temp(self): """ 清理生成的中间文件 + :return: """ if os.path.exists(self._work_dir): shutil.rmtree(self._work_dir) + shutil.rmtree(self.rpmbuild_dir) diff --git a/src/ac/framework/ac.yaml b/src/ac/framework/ac.yaml index 991cb07..7820bb4 100644 --- a/src/ac/framework/ac.yaml +++ b/src/ac/framework/ac.yaml @@ -30,7 +30,9 @@ src-openeuler: commit_msg: exclude: True source_consistency: - exclude: True + hint: check_consistency + module: source_consistency.check_consistency + entry: CheckSourceConsistency openeuler: spec: exclude: True -- Gitee