From f14792798f611afdb2fab19872ee10d2f60726b9 Mon Sep 17 00:00:00 2001 From: yang-yulong-007 <528553518@qq.com> Date: Wed, 12 Apr 2023 10:30:53 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Provides=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/obsoletes_rpms/conf/__init__.py | 0 .../{tools => conf}/logger.conf | 0 .../conf/rename_repository.json | 5 + .../obsoletes_rpms/retrieve_obsolete_rpms.py | 160 +++++++++++++----- tools/obsoletes_rpms/tools/logger.py | 2 +- 5 files changed, 122 insertions(+), 45 deletions(-) create mode 100644 tools/obsoletes_rpms/conf/__init__.py rename tools/obsoletes_rpms/{tools => conf}/logger.conf (100%) create mode 100644 tools/obsoletes_rpms/conf/rename_repository.json diff --git a/tools/obsoletes_rpms/conf/__init__.py b/tools/obsoletes_rpms/conf/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/obsoletes_rpms/tools/logger.conf b/tools/obsoletes_rpms/conf/logger.conf similarity index 100% rename from tools/obsoletes_rpms/tools/logger.conf rename to tools/obsoletes_rpms/conf/logger.conf diff --git a/tools/obsoletes_rpms/conf/rename_repository.json b/tools/obsoletes_rpms/conf/rename_repository.json new file mode 100644 index 0000000..c52cc58 --- /dev/null +++ b/tools/obsoletes_rpms/conf/rename_repository.json @@ -0,0 +1,5 @@ +{ + "scons": "python-scons", + "fonttools": "python-fonttools", + "pyyaml": "PyYAML" +} \ No newline at end of file diff --git a/tools/obsoletes_rpms/retrieve_obsolete_rpms.py b/tools/obsoletes_rpms/retrieve_obsolete_rpms.py index eea1db0..541ac36 100644 --- a/tools/obsoletes_rpms/retrieve_obsolete_rpms.py +++ b/tools/obsoletes_rpms/retrieve_obsolete_rpms.py @@ -17,6 +17,7 @@ """ import argparse import csv +import json import logging import re import shutil @@ -26,11 +27,12 @@ import sys from multiprocessing import Pool, cpu_count import os.path from urllib import request +from urllib.error import HTTPError from urllib.parse import urljoin import pandas as pd import wget -from git import Repo +from git import Repo, GitCommandError from tools.logger import init_logger from tools.shell_cmd import shell_cmd @@ -70,7 +72,7 @@ class RetrieveObsoleteRpms(object): stdin=stdin.stdout, stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT) - p.communicate() + p.communicate(timeout=600) @staticmethod def combine_single_result(results, src_base, bin_rpm, category): @@ -90,18 +92,36 @@ class RetrieveObsoleteRpms(object): return component_results @staticmethod - def get_obsoltes_rpms(spec_path, src_name): + def obtain_spec_file(dir_spec, src_name): + for file in os.listdir(dir_spec): + if file.endswith(".spec"): + return os.path.join(dir_spec, file) + + logger.warning(f"{src_name} spec file is not exists.") + return "" + + @staticmethod + def obtain_repository_name(src_name): + rename_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "conf/rename_repository.json")) + with open(rename_path, "r") as rf: + rename_repository = json.load(rf) + src_repo_name = rename_repository.get(src_name, src_name) + ".git" + + return src_repo_name + + @staticmethod + def prase_spec_with_plag(spec_path, src_name, flag): """ prase obsolete rpms from download spec file. @param spec_path: path of spec file. @param src_name: source rpm name. @return: """ - obsolete_rpms = [] - if not os.path.exists(spec_path): - return {} + obtain_results = [] + if not spec_path: + return obtain_results - cmd = f"rpmspec -q --obsoletes {spec_path}" + cmd = f"rpmspec -q --{flag.lower()} {spec_path}" code, out, err = shell_cmd(cmd.split()) if not code: if err: @@ -112,30 +132,30 @@ class RetrieveObsoleteRpms(object): continue symbol = re.search(r"[><=]=?", line) if symbol: - rpm_name = line.split(symbol.group())[0].strip() - obsolete_rpms.append(rpm_name) + tar_name = line.split(symbol.group())[0].strip() + obtain_results.append(tar_name) else: - obsolete_rpms.append(line.strip()) + obtain_results.append(line.strip()) else: - logger.debug(f"{spec_path} not found obsolete rpm.") + logger.debug(f"{spec_path} not found {flag}.") else: - logger.warning(f"Prase spec Error: {spec_path}") - with open(spec_path, "r") as s_f: - content = s_f.read() + logger.warning(f"Prase spec Error: {cmd}") + with open(spec_path, "r") as spec_f: + content = spec_f.read() name_version_pat = r"(\S+)\s([><=]=?)\s(\S+)" for line in content.split('\n'): - pat = r"^Obsoletes:" + pat = r"^" + flag + ":" if re.match(pat, line): - full_obsoletes_rpm = re.sub(pat, '', line).strip() - match_result = re.findall(name_version_pat, full_obsoletes_rpm) + line_strips = re.sub(pat, '', line).strip() + match_result = re.findall(name_version_pat, line_strips) if match_result: - for obsoletes_rpm in match_result: - obsolete_rpms.append(obsoletes_rpm[0].replace("%{name}", src_name)) + for single_tar in match_result: + obtain_results.append(single_tar[0].replace("%{name}", src_name)) else: - component_rpms = [rpm.replace("%{name}", src_name) for rpm in full_obsoletes_rpm.split()] - obsolete_rpms.extend(component_rpms) + component_tar = [single.replace("%{name}", src_name) for single in line_strips.split()] + obtain_results.extend(component_tar) - return obsolete_rpms + return obtain_results @staticmethod def write_to_csv_report(results, branch, model, work_dir): @@ -158,6 +178,45 @@ class RetrieveObsoleteRpms(object): Resolve rpm packages removed from older versions. @param all_rpm_report: oecp generated all-rpm-report.csv @return -> Dict: Deleted src rpm mapping to binary rpms. + + format eg: + { + "cookxml": { + "result": [ + { + "src_rpm": "cookxml-x.x.x-x.oe1.src.rpm", + "binary_rpm": "cookxml-x.x.x-x.oe1.noarch.rpm", + "category": "src_detete" + } + ], + "src_update": "" + }, + "boost": { + "result": [ + { + "src_rpm": "boost-x.x.x-x.oe1.src.rpm", + "binary_rpm": "boost-build-x.x.x-x.oe1.noarch.rpm", + "category": "bin_rpm_detete" + }, + { + "src_rpm": "boost-x.x.x-x.oe1.src.rpm", + "binary_rpm": "boost-doctools-x.x.x-x.oe1.aarch64.rpm", + "category": "bin_rpm_detete" + }, + { + "src_rpm": "boost-x.x.x-x.oe1.src.rpm", + "binary_rpm": "boost-examples-x.x.x-x.oe1.noarch.rpm", + "category": "bin_rpm_detete" + } + ... + ], + "src_update": "boost-x.x.x-x.oe2203sp1.src.rpm" + } + + ... + + } + """ results = {} df = pd.read_csv(all_rpm_report) @@ -183,13 +242,15 @@ class RetrieveObsoleteRpms(object): }) if delete_rpm not in set(all_new_src_name.keys()): for rpm in bin_rpms['rpms']: - self.combine_single_result(results[delete_rpm].get("result"), src_full_name, rpm, 'src_detete') + self.combine_single_result(results.get(delete_rpm).get("result"), src_full_name, rpm, 'src_detete') else: for rpm in bin_rpms['rpms']: if self.get_rpm_name(rpm) in all_new_bin_name: - self.combine_single_result(results[delete_rpm].get("result"), src_full_name, rpm, 'arch_change') + self.combine_single_result(results.get(delete_rpm).get("result"), src_full_name, rpm, + 'arch_change') else: - self.combine_single_result(results[delete_rpm].get("result"), src_full_name, rpm, 'bin_rpm_detete') + self.combine_single_result(results.get(delete_rpm).get("result"), src_full_name, rpm, + 'bin_rpm_detete') logger.info(f"total retrieve deleted rpms belong to {len(results)} src rpm.") @@ -216,55 +277,66 @@ class RetrieveObsoleteRpms(object): def check_gitee_spec_file(self, work_dir, src_name, content, branch): rpms = content["result"] - git_src = src_name + ".git" - base_url = urljoin(self.src_openeuler, src_name) - git_url = urljoin(self.src_openeuler, git_src) + src_repo_name = self.obtain_repository_name(src_name) + base_url = urljoin(self.src_openeuler, src_repo_name) + src_repo_url = urljoin(self.src_openeuler, src_repo_name) dir_path = os.path.join(work_dir, src_name) + os.makedirs(dir_path) try: response = request.urlopen(base_url) if response.getcode() == 200: - logger.info(f"download and prase {src_name} spec file") - Repo.clone_from(git_url, to_path=dir_path, branch=branch) - except Exception: - logger.debug(f"Clone {src_name} failed, maybe not exist branch-{branch}.") + Repo.clone_from(src_repo_url, to_path=dir_path, branch=branch) + logger.info(f"Download and prase {src_name} spec file") + else: + logger.warning(f"{src_name} repository is inaccessible: {base_url}") + except GitCommandError: + logger.warning(f"{src_name} Remote branch {branch} not found in upstream origin") + except HTTPError as err: + logger.error(f"{err}: {base_url}") finally: - component_rpms = self.search_obsolete_flag(dir_path, src_name, rpms) + component_rpms = self.combine_prase_result(dir_path, src_name, rpms) return component_rpms - def wget_repo_src_rpms(self, work_dir, content, branch): + def wget_repo_src_rpms(self, work_dir, src_name, content, branch): rpms = content["result"] url_source_repo = f"https://repo.huaweicloud.com/openeuler/{branch}/source/Packages/" src_rpm_full_name = content.get('src_update') - src_name = self.get_rpm_name(src_rpm_full_name) src_url = os.path.join(url_source_repo, src_rpm_full_name) download_dir = os.path.join(work_dir, src_name) + os.makedirs(download_dir) try: if src_rpm_full_name: - os.makedirs(download_dir) - logger.info(f"download and prase {src_name} spec file") src_path = wget.download(src_url, out=download_dir, bar=None) + logger.info(f"Download and prase {src_name} spec file") os.chdir(download_dir) self.perform_cpio(src_path) os.chdir(os.getcwd()) + else: + logger.warning(f"srpm {src_name} is deleted at new version") except Exception as err: logger.error(err) finally: - component_rpms = self.search_obsolete_flag(download_dir, src_name, rpms) + component_rpms = self.combine_prase_result(download_dir, src_name, rpms) return component_rpms - def search_obsolete_flag(self, dir_path, src_name, rpms): + def combine_prase_result(self, dir_path, src_name, rpms): component_rpms = [] - spec_file = os.path.join(dir_path, f"{src_name}.spec") - obsolete_rpms = self.get_obsoltes_rpms(spec_file, src_name) + spec_file = self.obtain_spec_file(dir_path, src_name) + obsolete_rpms = self.prase_spec_with_plag(spec_file, src_name, "Obsoletes") + provides = self.prase_spec_with_plag(spec_file, src_name, "Provides") for rpm in rpms: # 可能存在没有该分支的spec文件，或spec中不存在Obsoletes标识 rpm.setdefault('obsoletes_flag', 'no exist') - rpm.setdefault('spec_obsoletes_rpms', ','.join(obsolete_rpms)) + rpm.setdefault('spec_obsoletes_rpms', ','.join(set(obsolete_rpms))) + rpm.setdefault('provide_flag', 'no exist') + rpm.setdefault('spec_provides', ','.join(set(provides))) rpm_name = self.get_rpm_name(rpm['binary_rpm']) - if obsolete_rpms and rpm_name in obsolete_rpms: + if rpm_name in obsolete_rpms: rpm['obsoletes_flag'] = 'exist' + if rpm_name in provides: + rpm['provide_flag'] = 'exist' component_rpms.append(rpm) return component_rpms @@ -289,7 +361,7 @@ class RetrieveObsoleteRpms(object): pool.apply_async(self.check_gitee_spec_file, (download_dir, src_name, content, branch), callback=component_results.extend) elif model == "os": - pool.apply_async(self.wget_repo_src_rpms, (download_dir, content, branch), + pool.apply_async(self.wget_repo_src_rpms, (download_dir, src_name, content, branch), callback=component_results.extend) else: logger.error(f"input model is error,please check it.") diff --git a/tools/obsoletes_rpms/tools/logger.py b/tools/obsoletes_rpms/tools/logger.py index 3690377..ae3ee5e 100644 --- a/tools/obsoletes_rpms/tools/logger.py +++ b/tools/obsoletes_rpms/tools/logger.py @@ -7,7 +7,7 @@ def init_logger(): 初始化logger :return: """ - conf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "logger.conf")) + conf_path = os.path.realpath(os.path.join(os.path.dirname(__file__), "../conf/logger.conf")) defaults = {'args': str(('obsolete.log', 'a+', 50 * 1024 * 1024, 5))} logging.config.fileConfig(conf_path, defaults=defaults) \ No newline at end of file -- Gitee