diff --git a/core/obs_package_admc.py b/core/obs_package_admc.py new file mode 100755 index 0000000000000000000000000000000000000000..b2365def0dc45c43d8c460433653fa349408f130 --- /dev/null +++ b/core/obs_package_admc.py @@ -0,0 +1,395 @@ +#!/bin/env python3 + +""" +created by: wangchong +date: 2020-10-26 +""" + +import os +import re +import sys +import git +import shutil +sys.path.append('../') +from common.log_obs import log + +class OBSManager(object): + """ + obs project package add delete modify check + """ + def __init__(self, giteeUserName, giteeUserPwd): + """ + obs_meta_path: obs_meta dir path + patch_file_path: diff_patch file path + """ + self.work_dir = "/jenkins_home/workspace/obs_meta_update/openeuler_jenkins" + self.obs_meta_path = os.path.join(self.work_dir, "obs_meta") + self.patch_file_path = os.path.join(self.work_dir, "diff_patch") + self.giteeUserName = giteeUserName + self.giteeUserPwd = giteeUserPwd + self.import_list = [[]] * 2000 + + def _pre_env(self): + """ + initialize the workdir + """ + if os.path.exists(self.work_dir): + shutil.rmtree(self.work_dir) + os.makedirs(self.work_dir) + os.chdir(self.work_dir) + + def _git_clone(self, git_house): + """ + git clone function + """ + if git_house == "obs_meta": + if os.path.exists(self.obs_meta_path): + shutil.rmtree(self.obs_meta_path) + git_url = "https://%s:%s@gitee.com/src-openeuler/obs_meta.git" % ( + self.giteeUserName, self.giteeUserPwd) + git.Repo.clone_from(url = git_url, to_path = self.obs_meta_path) + os.chdir(self.obs_meta_path) + os.system("git diff --name-status HEAD~1 HEAD~0 > %s" + % self.patch_file_path) + if git_house == "community": + community_path = os.path.join(self.work_dir, "community") + if os.path.exists(community_path): + shutil.rmtree(community_path) + git_url = "https://%s:%s@gitee.com/openeuler/community.git" % ( + self.giteeUserName, self.giteeUserPwd) + git.Repo.clone_from(url = git_url, to_path = community_path) + os.chdir(self.work_dir) + + def _add_pkg(self, proj, pkg, branch_name): + """ + add project package + """ + os.chdir(self.work_dir) + if os.path.exists(proj): + shutil.rmtree(proj) + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: + os.system("osc co %s %s &>/dev/null" % (proj, pkg)) + else: + os.system("osc co %s `osc ls %s | sed -n '1p'` &>/dev/null" % (proj, proj)) + pkg_path = os.path.join(self.obs_meta_path, '%s/%s/%s' % (branch_name, proj, pkg)) + os.chdir(proj) + if os.path.exists(pkg): + os.system("cp -rf %s ." % pkg_path) + cmd = "osc status | grep ^? | awk '{print 2}'" + new_file = os.popen(cmd).read() + if len(new_file): + os.system("osc add %s" % new_file) + else: + os.system("cp -rf %s ." % pkg_path) + rm_dir = os.path.join('%s/.osc' % pkg) + if os.path.exists(rm_dir): + os.system("rm -rf %s" % rm_dir) + os.system("osc add %s" % pkg) + new_file = os.popen("osc status").read() + if len(new_file): + os.system("osc ci -m 'add by %s'" % self.giteeUserName) + os.chdir(self.work_dir) + + def _add_pkg_service(self, proj, pkg, branch_name): + """ + write and push _service file in obs_meta + """ + proj_path = os.path.join(self.obs_meta_path, branch_name, proj) + if not os.path.exists(proj_path): + log.warning("obs_meta do not have %s %s" % (branch_name, proj)) + return -1 + os.chdir(proj_path) + if not os.path.exists(pkg): + os.mkdir(pkg) + os.chdir(pkg) + f = open("_service", 'w') + f.write('\n') + f.write(' \n') + f.write(' repo\n') + if branch_name == "master": + f.write(' next/openEuler/%s\n' % pkg) + else: + f.write(' next/%s/%s\n' % (branch_name, pkg)) + f.write(' \n') + f.write('\n') + f.close() + os.chdir("%s/%s/%s" % (self.obs_meta_path, branch_name, proj)) + os.system("git add %s" % pkg) + os.system("git commit -m 'add %s _service file by %s'" % (pkg, self.giteeUserName)) + os.system("git push") + + def _del_pkg(self, proj, pkg): + """ + delete the project package + """ + os.chdir(self.work_dir) + proj_path = os.path.join(self.work_dir, proj) + if os.path.exists(proj_path): + shutil.rmtree(proj) + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: + os.system("osc co %s %s &>/dev/null" % (proj, pkg)) + os.chdir(proj) + os.system("osc rm %s" % pkg) + os.system("osc ci -m 'delete by %s'" % self.giteeUserName) + else: + log.warning("obs %s %s not found" % (proj, pkg)) + return -1 + os.chdir(self.work_dir) + + def _del_obs_pkg_service(self, proj, pkg): + """ + delete the obs project package service file + """ + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) != 0: + log.warning("obs %s %s not found" % (proj, pkg)) + return -1 + os.chdir(self.work_dir) + proj_path = os.path.join(self.work_dir, proj) + pkg_path = os.path.join(proj_path, pkg) + if os.path.exists(proj_path): + shutil.rmtree(proj) + os.system("osc co %s %s &>/dev/null" % (proj, pkg)) + os.chdir(pkg_path) + os.system("test -f _service && osc rm _service") + os.system("osc ci -m 'delete by %s'" % self.giteeUserName) + os.chdir(self.work_dir) + + def _del_meta_pkg_service(self, branch, proj, pkg): + """ + delete the obs_meta project pkg service file + """ + if not os.path.exists(os.path.join(self.obs_meta_path, branch, proj, pkg)): + log.warning("obs_meta %s %s %s not found" % (branch, proj, pkg)) + return -1 + proj_path = os.path.join(self.obs_meta_path, branch, proj) + pkg_path = os.path.join(proj_path, pkg) + os.chdir(proj_path) + if os.path.exists(pkg_path): + os.system("rm -rf %s" % pkg) + os.system("git add -A && git commit -m 'delete by %s'" % self.giteeUserName) + os.system("git push") + os.chdir(self.work_dir) + + def _modify_pkg_service(self, proj, pkg, branch_name): + """ + change the service file for the package + """ + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) != 0: + log.warning("%s %s not found" % (proj, pkg)) + return -1 + os.chdir(self.work_dir) + proj_path = os.path.join(self.work_dir, proj) + pkg_path = os.path.join(proj_path, pkg) + service_file_path = os.path.join(self.obs_meta_path, '%s/%s/%s/_service' % (branch_name, proj, pkg)) + if os.path.exists(proj_path): + shutil.rmtree(proj) + os.system("osc co %s %s &>/dev/null" % (proj, pkg)) + os.system("cp -f %s %s" % (service_file_path, pkg_path)) + os.chdir(pkg_path) + os.system("osc add _service") + os.system("osc ci -m 'modify by %s'" % self.giteeUserName) + os.chdir(self.work_dir) + + def _modify_pkg_meta(self, proj, pkg, branch_name): + """ + change the package of the meta + """ + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) != 0: + log.warning("%s %s not found" % (proj, pkg)) + return -1 + os.chdir(self.work_dir) + file_path = os.path.join(self.obs_meta_path, "%s/%s/%s/.osc/_meta" % (branch_name, proj, pkg)) + cmd = "osc meta pkg %s %s --file=%s | grep ^Done." % (proj, pkg, file_path) + if os.system(cmd) != 0: + log.error("%s/%s/.osc/_meta deal error" % (proj, pkg)) + + def _change_pkg_prj(self, proj, new_proj, pkg, branch_name): + """ + change the package of the project + """ + self._del_pkg(proj, pkg) + self._add_pkg(new_proj, pkg, branch_name) + + def _parse_git_log(self, line): + """ + deal diff_patch line mesg + """ + new_proj = '' + new_file_path = '' + log_list = list(line.split()) + temp_log_type = log_list[0] + file_path = log_list[1] + if len(log_list) == 3: + new_file_path = list(line.split())[2] + branch_name, proj, pkg, file_name = str(file_path).split('/') + if len(new_file_path) != 0: + new_proj = new_file_path.split('/')[1] + log_type = "Change-pkg-prj" + elif file_name == "_meta": + if temp_log_type == "A" or temp_log_type == "M": + log_type = "Mod-pkg-meta" + else: + log.error("%s failed" % line) + elif file_name == "_service": + if temp_log_type == "A": + log_type = "Add-pkg" + elif temp_log_type == "D": + pkg_path = os.path.join(self.obs_meta_path, '%s/%s/%s' % (branch_name, proj, pkg)) + if os.path.exists(pkg_path): + log_type = "Del-pkg-service" + else: + log_type = "Del-pkg" + line = line.replace('_service', '').replace('\n', '') + pattern_message = line.replace(' ', '\t').replace('/', '\\/') + os.system("sed -i '/%s/d' %s" % (pattern_message, self.patch_file_path)) + elif temp_log_type == "M": + log_type = "Mod-pkg-service" + else: + log.error("%s failed" % line) + else: + log.error("%s failed" % line) + mesg_list = [log_type, branch_name, proj, pkg, new_proj] + return mesg_list + + def _deal_some_param(self): + """ + deal with some data and relation + """ + pattern_string = ['.meta', '.prjconf', '/_service', '/_meta'] + num = 0 + for pattern in pattern_string: + data = open(self.patch_file_path, 'r') + for line in data: + cmd1 = 'echo "%s" | grep "%s$"' % (line.strip(), pattern) + if os.popen(cmd1).read(): + log_type, branch_name, proj, pkg, new_prj = self._parse_git_log(line.strip('\n')) + exist_flag = 0 + prj_list = ['openEuler:Mainline', 'openEuler:Factory', + 'openEuler:Epol', 'openEuler:Extras', 'bringInRely'] + for p in prj_list: + cmd3 = "osc ls %s 2>&1 | grep -q -Fx %s" % (p, pkg) + if os.system(cmd3) == 0: + if p == proj: + log.info("package %s hava existed in obs project %s" % (pkg, p)) + exist_flag = 1 + self.import_list[num] = [log_type, branch_name, proj, pkg, new_prj, exist_flag] + num += 1 + else: + continue + data.close() + + def obs_pkg_admc(self): + """ + obs project package add, delete, modify, check + """ + self._pre_env() + self._git_clone("obs_meta") + self._deal_some_param() + for i in range(len(self.import_list)): + if len(self.import_list[i]): + log_type, branch_name, proj, pkg, new_proj, exist_flag = self.import_list[i] + log.info("log_type:%s branch_name:%s proj:%s pkg:%s new_proj:%s exist_flag:%s" + % (log_type, branch_name, proj, pkg, new_proj, exist_flag)) + if log_type == "Add-pkg": + if exist_flag == 0: + self._add_pkg(proj, pkg, branch_name) + elif log_type == "Del-pkg": + self._del_pkg(proj, pkg) + elif log_type == "Del-pkg-service": + self._del_obs_pkg_service(proj, pkg) + elif log_type == "Mod-pkg-service": + self._modify_pkg_service(proj, pkg, branch_name) + elif log_type == "Mod-pkg-meta": + self._modify_pkg_meta(proj, pkg, branch_name) + elif log_type == "Change-pkg-prj": + self._change_pkg_prj(proj, new_proj, pkg, branch_name) + else: + return 0 + + def check_obs_pkg(self): + """ + check the obs project and operate according to the src-openeuler.yaml file + """ + self._pre_env() + self._git_clone("community") + self._git_clone("obs_meta") + os.chdir(self.work_dir) + yaml_path = os.path.join(self.work_dir, "community/repository") + f1 = open("%s/src-openeuler.yaml" % yaml_path, 'r') + yaml_dict = {} + meta_bp_dict = {} + pkg_branch_dict = {} + branch_proj_dict = {'master':'openEuler:Factory', + 'openEuler-20.09':'openEuler:20.09', + 'openEuler-20.03-LTS':'openEuler:20.03:LTS', + 'openEuler-20.03-LTS-Next':'openEuler:20.03:LTS:Next', + } + for line in f1: + if re.search("- name:", line): + pkg_name = line.split()[2] + if re.search(" - ", line): + branch = line.split()[1] + if branch != "openEuler1.0-base" and branch != "openEuler1.0": + yaml_dict.setdefault(pkg_name,[]).append(branch) + f1.close() + os.chdir(self.obs_meta_path) + cmd = "find | grep _service | grep -Ev 'RISC-V|OBS_PRJ_meta|openEuler-EPOL-LTS|openEuler-20.03-SP1' | \ + awk -F '/' '{print $2,$3,$(NF-1)}' | sort | uniq > %s/res.txt" % self.work_dir + while os.system(cmd) != 0: + continue + os.chdir(self.work_dir) + f2 = open("%s/res.txt" % self.work_dir, 'r') + for line in f2: + br = line.strip().split()[0] + proj = line.strip().split()[1] + name = line.strip().split()[2] + meta_bp_dict.setdefault(br,[]).append(proj) + pkg_branch_dict.setdefault(name,[]).append(br) + f2.close() + log.info("check BEGIN") + log.info("check stage 1:") + for pkg, branch in yaml_dict.items(): + if pkg in pkg_branch_dict: + branch.sort() + pkg_branch_dict[pkg].sort() + if branch == pkg_branch_dict[pkg]: + continue + else: + diff_add_br = set(branch).difference(set(pkg_branch_dict[pkg])) + for diff in diff_add_br: + self._add_pkg_service(branch_proj_dict[diff], pkg, diff) + self._add_pkg(branch_proj_dict[diff], pkg, diff) + else: + for need_add_br in yaml_dict[pkg]: + self._add_pkg_service(branch_proj_dict[need_add_br], pkg, need_add_br) + self._add_pkg(branch_proj_dict[need_add_br], pkg, need_add_br) + log.info("check stage 2:") + for pkg, branch in pkg_branch_dict.items(): + if pkg in yaml_dict: + branch.sort() + yaml_dict[pkg].sort() + if branch == yaml_dict[pkg]: + continue + else: + diff_del_br = set(branch).difference(set(yaml_dict[pkg])) + for diff in diff_del_br: + for proj in meta_bp_dict[diff]: + self._del_meta_pkg_service(diff, proj, pkg) + self._del_pkg(proj, pkg) + else: + for need_del_br in pkg_branch_dict[pkg]: + for proj in meta_bp_dict[need_del_br]: + self._del_meta_pkg_service(need_del_br, proj, pkg) + self._del_pkg(proj, pkg) + os.remove("%s/res.txt" % self.work_dir) + log.info("check END") + + +if __name__ == "__main__": + """ + sys.argv[1]: git user name + sys.argv[2]: git user pwd + """ + om = OBSManager(sys.argv[1], sys.argv[2]) + om.obs_pkg_admc() + om.check_obs_pkg()