diff --git a/script/tools/check_dep.py b/script/tools/check_dep.py
index 2b46009de8d82c4c399b3e2721a9efd0044a294c..1b34c772db647c1e03511c1f2c6f574c4d96a2a5 100644
--- a/script/tools/check_dep.py
+++ b/script/tools/check_dep.py
@@ -186,13 +186,13 @@ def set_exclude_pkg_all_rpms():
"""
get all rpms of exclude rpm list and source rpms
"""
- print("=========== start search all rpms of exclude rpm list ===========")
with open(args.exclude_rpm_list_file, "r") as f:
file_content = f.read().strip().splitlines()
cmd = f"rm -f {args.final_exclude_rpm_list_file} {args.final_source_exclude_rpm_list_file} && touch {args.final_exclude_rpm_list_file} {args.final_source_exclude_rpm_list_file}"
if os.system(cmd) == 0:
pass
if file_content:
+ print("============ start search all rpms of exclude rpm list ===========")
pkg_rpms_list = []
cmd = "uname -m"
arch = os.popen(cmd).read().strip()
@@ -203,11 +203,17 @@ def set_exclude_pkg_all_rpms():
for pkg in pkglist:
executor.submit(get_pkg_rpms, pkg, arch, pkg_rpms_list)
final_rpms_list = []
+ rpms_reason_dict = {}
for rpms in pkg_rpms_list:
+ err_rpm_list = []
for rpm in rpms:
if rpm in file_content:
if rpm not in final_rpms_list:
final_rpms_list.extend(rpms)
+ err_rpm_list.append(rpm)
+ if err_rpm_list:
+ rpms_reason_dict.setdefault("install_problem", []).extend(err_rpm_list)
+ rpms_reason_dict.setdefault("other_rpm_install_problem", []).extend(list(set(rpms) - set(err_rpm_list)))
if final_rpms_list:
f1 = open(args.final_exclude_rpm_list_file, "w")
f2 = open(args.final_source_exclude_rpm_list_file, "w")
@@ -220,10 +226,27 @@ def set_exclude_pkg_all_rpms():
f1.write("\n")
f1.close()
f2.close()
+ print("二进制包:")
print(os.popen(f"cat {args.final_exclude_rpm_list_file}").read())
+ print("源码包:")
print(os.popen(f"cat {args.final_source_exclude_rpm_list_file}").read())
- print("============ end search all rpms of exclude rpm list ============")
-
+ print("============ end search all rpms of exclude rpm list ============")
+ print("============ start reason for exclude rpm list ============")
+ if rpms_reason_dict:
+ for key, value in rpms_reason_dict.items():
+ for rpm in value:
+ if rpm.endswith(".src.rpm"):
+ rpm_name = rpm.rsplit("-", 2)[0]
+ rpm_type = "源码包\t"
+ else:
+ rpm_name = rpm
+ rpm_type = "二进制包"
+ if key == "install_problem":
+ msg = "校验安装失败\t\t\t\t%s\t\t%s" % (rpm_type, rpm_name)
+ if key == "other_rpm_install_problem":
+ msg = "对应软件包其它二进制安装失败\t\t%s\t\t%s" % (rpm_type, rpm_name)
+ print(msg)
+ print("============ end reason for exclude rpm list ==============")
par = argparse.ArgumentParser()
par.add_argument("-d", "--dest_rpm_path", help="path for rpm", required=True)
diff --git a/script/tools/check_rpm_repo_complete.py b/script/tools/check_rpm_repo_complete.py
new file mode 100644
index 0000000000000000000000000000000000000000..b834b4067d264d1d8fa5498ea6fca685c9bf73da
--- /dev/null
+++ b/script/tools/check_rpm_repo_complete.py
@@ -0,0 +1,180 @@
+#!/bin/env python3
+
+import os
+import sys
+import yaml
+import argparse
+import shutil
+from concurrent.futures import ThreadPoolExecutor
+
+
+par = argparse.ArgumentParser()
+par.add_argument("-c", "--config", help="config file for repo", default=None, required=True)
+par.add_argument("-r", "--repo", help="name of repo", default=None, required=True)
+par.add_argument("-p", "--project", help="name of project", default=None, required=True)
+par.add_argument("-f", "--logfile", help="not in repo rpm list file", default=None, required=True)
+args = par.parse_args()
+
+
+def git_clone(git_url, repo_name, branch):
+ """
+ git clone gitee repo
+ """
+ repo_path = os.path.join(os.getcwd(), repo_name)
+ if os.path.exists(repo_path):
+ shutil.rmtree(repo_path)
+ cmd = "git clone --depth 1 %s -b %s" % (git_url, branch)
+ if os.system(cmd) != 0:
+ print("Git clone %s failed!" % repo_name)
+ sys.exit(1)
+ else:
+ print("Git clone %s success!" % repo_name)
+ return repo_path
+
+def get_release_pkg_list():
+ """
+ get release pkg list
+ """
+ release_pkglist = []
+ repo_name = "release-management"
+ git_url = f"https://gitee.com/openeuler/{repo_name}.git"
+ repo_path = git_clone(git_url, repo_name, "master")
+ if args.project == "openEuler:Mainline":
+ branch = "master"
+ elif args.project.endswith(":Epol"):
+ branch = args.project.replace(":Epol", "").replace(":", "-")
+ path_name = ["epol"]
+ else:
+ branch = args.project.replace(":", "-")
+ path_name = ["baseos", "everything-exclude-baseos"]
+ for name in path_name:
+ yaml_path = os.path.join(repo_path, branch, name, "pckg-mgmt.yaml")
+ if os.path.exists(yaml_path):
+ with open(yaml_path, "r", encoding="utf-8") as f:
+ result = yaml.load(f, Loader=yaml.FullLoader)
+ for pckg in result['packages']:
+ if pckg['name'] not in release_pkglist:
+ release_pkglist.append(pckg['name'])
+ shutil.rmtree(repo_path)
+ return release_pkglist
+
+def get_exclude_rpm_list():
+ """
+ get oemaker exclude rpm list
+ """
+ exclude_rpmlist = []
+ if args.project == "openEuler:Mainline" or args.project == "openEuler:Epol":
+ branch = "master"
+ elif args.project.endswith(":Epol"):
+ branch = args.project.replace(":Epol", "").replace(":", "-")
+ else:
+ branch = args.project.replace(":", "-")
+ repo_name = "oemaker"
+ git_url = f"https://gitee.com/src-openeuler/{repo_name}.git"
+ repo_path = git_clone(git_url, repo_name, branch)
+ cmd = "xmllint --xpath \"//packagelist[@type='exclude']/node()\" %s/rpmlist.xml \
+ | grep packagereq | cut -d '>' -f 2 | cut -d '<' -f 1" % repo_path
+ ret = os.popen(cmd).read().split('\n')
+ exclude_rpmlist = [ x for x in ret if x != "" ]
+ shutil.rmtree(repo_path)
+ print("oemaker rpmlist.xml exclude rpm:%s" % exclude_rpmlist)
+ return exclude_rpmlist
+
+def get_repo_rpm_list():
+ """
+ get repo all rpms
+ """
+ tmp_path = "/tmp/_repo_rpm"
+ if os.path.exists(tmp_path):
+ shutil.rmtree(tmp_path)
+ cmd = "yum list --installroot=%s --available -c %s --repo %s | grep %s | awk '{print $1,$2}'" % (tmp_path, args.config, args.repo, args.repo)
+ output = os.popen(cmd).read().split('\n')
+ result = [ x for x in output if x != "" ]
+ if result:
+ del(result[0])
+ repo_rpm_list = []
+ arch_list = [".aarch64", ".x86_64", ".noarch", ".src"]
+ for line in result:
+ tmp = line.split()
+ for arch in arch_list:
+ if arch in tmp[0]:
+ new_tmp = tmp[0].split(arch)
+ name = new_tmp[0]
+ if ":" in tmp[1]:
+ version = tmp[1].split(":")[1]
+ else:
+ version = tmp[1]
+ rpm_name = f"{name}-{version}{arch}.rpm"
+ break
+ if rpm_name not in repo_rpm_list:
+ repo_rpm_list.append(rpm_name)
+ return repo_rpm_list
+
+def get_pkg_rpms(pkg, arch, pkg_rpm_list):
+ """
+ get a package all rpm
+ """
+ cmd = f"osc ls -b {args.project} {pkg} standard_{arch} {arch} 2>/dev/null | grep rpm"
+ rpm_list = os.popen(cmd).read().split()
+ new_rpm_list = [rpm for rpm in rpm_list if rpm != '']
+ if new_rpm_list:
+ pkg_rpm_list.extend(new_rpm_list)
+
+def get_release_all_pkg_rpms(release_pkg_list):
+ """
+ get rpms of all pkg
+ """
+ pkg_rpm_list = []
+ cmd = "arch"
+ arch = os.popen(cmd).read().strip()
+ with ThreadPoolExecutor(100) as executor:
+ for pkg in release_pkg_list:
+ executor.submit(get_pkg_rpms, pkg, arch, pkg_rpm_list)
+ return pkg_rpm_list
+
+def delete_exclude_rpm(not_in_repo_rpm):
+ """
+ delete exclude rpm
+ """
+ final_rpm = []
+ if not args.project.endswith(":Epol"):
+ exclude_rpmlist = get_exclude_rpm_list()
+ if exclude_rpmlist:
+ for repo_rpm in not_in_repo_rpm:
+ rpm_name = repo_rpm.rsplit("-", 2)[0]
+ if rpm_name not in exclude_rpmlist:
+ final_rpm.append(repo_rpm)
+ else:
+ final_rpm = not_in_repo_rpm
+ return final_rpm
+
+def write_file(result):
+ if os.path.exists(args.logfile):
+ with open(args.logfile, "w") as f:
+ for line in result:
+ f.write(line)
+ f.write("\n")
+
+def check_repo_complete():
+ """
+ check project all pkg rpm equal repo all rpm
+ """
+ print("========== start check release package rpm in repo ==========")
+ release_pkg_list = get_release_pkg_list()
+ all_pkg_rpm_list = get_release_all_pkg_rpms(release_pkg_list)
+ print("all_pkg_rpm_list:%s" % all_pkg_rpm_list)
+ repo_rpm_list = get_repo_rpm_list()
+ print("repo_rpm_list:%s" % repo_rpm_list)
+ not_in_repo_rpm = list(set(all_pkg_rpm_list) - set(repo_rpm_list))
+ if not_in_repo_rpm:
+ final_result = delete_exclude_rpm(not_in_repo_rpm)
+ print("[FAILED] some package rpm not in repo without exclude rpm")
+ if final_result:
+ write_file(final_result)
+ print("\n".join(final_result))
+ else:
+ print("[SUCCESS] all package rpm in repo")
+ print("========== end check release package rpm in repo ==========")
+
+
+check_repo_complete()
diff --git a/script/tools/package_install_problem_report.py b/script/tools/package_install_problem_report.py
index da5f3e24f680b492407c0f27ac0eb4947d2262aa..4a70c2f0c62f6ed86618fea0d6c0b17ee1073590 100644
--- a/script/tools/package_install_problem_report.py
+++ b/script/tools/package_install_problem_report.py
@@ -16,6 +16,7 @@
import os
import re
+import sys
import shutil
import logging
import argparse
@@ -90,7 +91,7 @@ def git_clone():
branch = args.branch
cmd = "git clone --depth 1 %s -b %s" % (git_url, branch)
if os.system(cmd) != 0:
- print("Git clone oemaker failed!")
+ log.error("Git clone oemaker failed!")
sys.exit(1)
return repo_path
@@ -150,6 +151,7 @@ def parse_msg(subjob_url_list, short_list, exclude_rpmlist, file_msg):
"""
final_result = {}
allpkgs = []
+ reason_msg = []
for surl, url in zip(short_list, subjob_url_list):
pkglist = []
flag = False
@@ -160,6 +162,7 @@ def parse_msg(subjob_url_list, short_list, exclude_rpmlist, file_msg):
flag = True
continue
elif "list end" in line:
+ flag = False
break
if flag:
if line and line not in exclude_rpmlist:
@@ -167,7 +170,21 @@ def parse_msg(subjob_url_list, short_list, exclude_rpmlist, file_msg):
if line not in allpkgs:
allpkgs.append(line)
final_result.setdefault(surl, pkglist)
+ for line in output:
+ line_msg = {}
+ if "start reason for" in line:
+ flag = True
+ continue
+ elif "end reason for" in line:
+ break
+ if flag and line:
+ tmp_data = line.replace('\t', ' ').split()
+ line_msg['reason'] = tmp_data[0]
+ line_msg['type'] = tmp_data[1]
+ line_msg['rpm_name'] = tmp_data[2]
+ reason_msg.append(line_msg)
log.info("Final cannot install rpm:%s" % allpkgs)
+ log.info("reason msg:%s" % reason_msg)
failed_pkg = []
for bin_rpm in allpkgs:
tmp = {}
@@ -178,9 +195,9 @@ def parse_msg(subjob_url_list, short_list, exclude_rpmlist, file_msg):
tmp['team'] = data[0]
tmp['people'] = data[1]
failed_pkg.append(tmp)
- return final_result, failed_pkg
+ return final_result, failed_pkg, reason_msg
-def write_email_message(final_result, failed_pkg, lastbuildnumber):
+def write_email_message(final_result, failed_pkg, lastbuildnumber, reason_msg):
"""
write message
"""
@@ -190,6 +207,7 @@ def write_email_message(final_result, failed_pkg, lastbuildnumber):
msg = ""
pkgs = ""
line = ""
+ reason_line = ""
main_job_url = init_url + "/check-rpm-install-dependence-" + args.branch + "/" + lastbuildnumber + "/console"
log.info(main_job_url)
for key,value in final_result.items():
@@ -209,6 +227,10 @@ def write_email_message(final_result, failed_pkg, lastbuildnumber):
pkgs = pkgs + """
| %s | %s | %s | %s |
""" % (tmp['bin_rpm'], tmp['source_rpm'], tmp['team'], tmp['people'])
+ for reason in reason_msg:
+ reason_line = reason_line + """
+ | %s | %s | %s |
+ """ % (reason['reason'], reason['rpm_name'], reason['type'])
msg = """
Hello:
根据OBS工程repo源检查软件包安装问题，其日志链接如下：
@@ -222,7 +244,11 @@ def write_email_message(final_result, failed_pkg, lastbuildnumber):
%s
这些问题会阻塞ISO的构建，请尽快解决，谢谢~^V^~!!!
- """ % (line, args.branch, pkgs)
+
+ """ % (line, args.branch, pkgs, reason_line)
return msg
def send_email(message):
@@ -251,7 +277,7 @@ else:
lastbuildnumber = args.buildnumber
job_url_list, short_job_url = get_subjob_url(lastbuildnumber)
file_msg = read_yaml()
-final_result, allrpm = parse_msg(job_url_list, short_job_url, exclude_rpmlist, file_msg)
-message = write_email_message(final_result, allrpm, lastbuildnumber)
+final_result, allrpm, reason_message = parse_msg(job_url_list, short_job_url, exclude_rpmlist, file_msg)
+message = write_email_message(final_result, allrpm, lastbuildnumber, reason_message)
if message:
send_email(message)