From f698622728e89fbb2524ceee6c05e43c2d2e80c3 Mon Sep 17 00:00:00 2001 From: wangchong1995924 <15229716099@163.com> Date: Mon, 26 Oct 2020 16:37:36 +0800 Subject: [PATCH 1/6] add obs_package_admc.py --- core/obs_package_admc.py | 255 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 255 insertions(+) create mode 100755 core/obs_package_admc.py diff --git a/core/obs_package_admc.py b/core/obs_package_admc.py new file mode 100755 index 0000000..dce797e --- /dev/null +++ b/core/obs_package_admc.py @@ -0,0 +1,255 @@ +#!/bin/env python3 + +import shutil +import sys +import git +import os +import re +sys.path.append('../') +from common.log_obs import log + +work_dir = "/jenkins_home/workspace/obs_meta_update/openeuler_jenkins" +obs_meta_path = os.path.join(work_dir, "obs_meta") +patch_file_path = os.path.join(work_dir, "diff_patch") +giteeUserName = "openeuler-ci-bot" + +# initialize the workdir +def pre_env(): + if os.path.exists(work_dir): + shutil.rmtree(work_dir) + os.makedirs(work_dir) + +# git clone +def git_clone(git_url, git_path): + if os.path.exists(git_path): + shutil.rmtree(git_path) + git.Repo.clone_from(url = git_url, to_path = git_path) + +# add project package +def add_pkg(proj, pkg, branch_name): + log.info("Begin add_pkg") + os.chdir(work_dir) + if os.path.exists(proj): + shutil.rmtree(proj) + if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: + os.system("osc co %s %s &>/dev/null" %(proj, pkg)) + else: + os.system("osc co %s `osc ls %s | sed -n '1p'` &>/dev/null" %(proj, proj)) + pkg_path = os.path.join(obs_meta_path, '%s/%s/%s' % (branch_name, proj, pkg)) + os.chdir(proj) + if os.path.exists(pkg): + os.system("cp -rf %s ." % pkg_path) + cmd = "osc status | grep ^? | awk '{print 2}'" + new_file = os.popen(cmd).read() + if len(new_file): + os.system("osc add %s" %new_file) + else: + os.system("cp -rf %s ." % pkg_path) + rm_dir = os.path.join('%s/.osc' % pkg) + if os.path.exists(rm_dir): + os.system("rm -rf %s" % rm_dir) + os.system("osc add %s" % pkg) + new_file = os.popen("osc status").read() + if len(new_file): + log.info("osc ci -m 'add by %s'" % giteeUserName) +# os.system("osc ci -m 'add by %s'" % giteeUserName) + os.chdir(work_dir) + log.info("END add_pkg") + +# use by check_pkg function +def add_pkg_service(proj, pkg, branch_name): + log.info("Begin add_pkg_service") + os.chdir("%s/%s/%s"% (obs_meta_path, branch_name, proj)) + os.system("git pull &>/dev/null") + if not os.path.exists(pkg): + os.mkdir(pkg) + os.chdir(pkg) + f = open("_service",'w') + f.write('\n') + f.write(' \n') + f.write(' repo\n') + if branch_name == "master": + f.write(' next/openEuler/%s\n'% pkg) + else: + f.write(' next/%s/%s\n'% (branch_name, pkg)) + f.write(' \n') + f.write('\n') + f.close() + os.chdir("%s/%s/%s"% (obs_meta_path, branch_name, proj)) + ''' + os.popen("git add %s" % pkg) + os.popen("git commit -m 'add %s file _service by %s'"% (pkg, giteeUserName)) + os.popen("git push") + ''' + add_pkg(proj, pkg, branch_name) + log.info("END add_pkg_service") + +# delete the project package +def del_pkg(proj, pkg): + log.info("Begin del_pkg") + os.chdir(work_dir) + proj_path = os.path.join(work_dir, proj) + if os.path.exists(proj_path): + shutil.rmtree(proj) + if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: + os.system("osc co %s %s &>/dev/null" %(proj, pkg)) + os.chdir(proj) + os.system("osc rm %s" % pkg) + log.info("osc ci -m 'delete by %s'" % giteeUserName) +# os.system("osc ci -m 'delete by %s'" % giteeUserName) + os.chdir(work_dir) + log.info("END del_pkg") + +# delete the service file for the package +def del_pkg_service(proj, pkg): + log.info("Begin del_pkg_service") + if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: + log.warning("%s %s not found" % (proj, pkg)) + return 0 + os.chdir(work_dir) + proj_path = os.path.join(work_dir, proj) + pkg_path = os.path.join(proj_path, pkg) + if os.path.exists(proj_path): + shutil.rmtree(proj) + os.system("osc co %s %s &>/dev/null" %(proj, pkg)) + os.chdir(pkg_path) + os.system("test -f _service && osc rm _service") + log.info("osc ci -m 'delete by %s'" % giteeUserName) +# os.system("osc ci -m 'delete by %s'" % giteeUserName) + os.chdir(work_dir) + log.info("END del_pkg_service") + +# change the service file for the package +def modify_pkg_service(proj, pkg, branch_name): + log.info("Begin modify_pkg_service") + if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: + log.info("%s %s not found" % (proj, pkg)) + os.chdir(work_dir) + proj_path = os.path.join(work_dir, proj) + pkg_path = os.path.join(proj_path, pkg) + service_file_path = os.path.join(obs_meta_path, '%s/%s/%s/_service' % (branch_name, proj, pkg)) + if os.path.exists(proj_path): + shutil.rmtree(proj) + os.system("osc co %s %s &>/dev/null" %(proj, pkg)) + os.system("cp -f %s %s" %(service_file_path, pkg_path)) + os.chdir(pkg_path) + os.system("osc add _service") + log.info("osc ci -m 'modify by %s'" % giteeUserName) +# os.system("osc ci -m 'modify by %s'" % giteeUserName) + os.chdir(work_dir) + log.info("END modify_pkg_service") + +# change the package of the meta +def modify_pkg_meta(proj, pkg, branch_name): + log.info("Begin modify_pkg_meta") + if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: + log.info("%s %s not found" % (proj, pkg)) + os.chdir(work_dir) + file_path = os.path.join(obs_meta_path, "%s/%s/%s/.osc/_meta" % (branch_name, proj, pkg)) + cmd = "osc meta pkg %s %s --file=%s | grep ^Done." % (proj, pkg, file_path) + if os.system(cmd) != 0: + log.info("%s/%s/.osc/_meta deal error" % (proj, pkg)) + log.info("END modify_pkg_meta") + +# change the package of the project +def change_pkg_prj(proj, new_proj, pkg, branch_name): + log.info("Begin change_pkg_prj") + del_pkg(proj, pkg) + add_pkg(new_proj, pkg, branch_name) + log.info("END change_pkg_prj") + +# deal diff_patch line mesg +def parse_git_log(line): + new_proj = '' + log_list = list(line.split()) + temp_log_type = log_list[0] + file_path = log_list[1] + if len(log_list) == 3: + new_file_path = list(line.split())[2] + else: + new_file_path = '' + branch_name, proj, pkg, file_name = str(file_path).split('/') + if len(new_file_path) != 0: + new_proj = new_file_path.split('/')[1] + log_type = "Change-pkg-prj" + elif file_name == "_meta": + if temp_log_type == "A" or temp_log_type == "M": + log_type = "Mod-pkg-meta" + else: + log.info("%s failed" % line) + elif file_name == "_service": + if temp_log_type == "A": + log_type = "Add-pkg" + elif temp_log_type == "D": + pkg_path = os.path.join(obs_meta_path, '%s/%s/%s' % (branch_name, proj, pkg)) + if os.path.exists(pkg_path): + log_type = "Del-pkg-service" + else: + log_type = "Del-pkg" + line = line.replace('_service','').replace('\n', '') + pattern_message = line.replace(' ', '\t').replace('/', '\\/') + os.system("sed -i '/%s/d' %s" % (pattern_message, patch_file_path)) + elif temp_log_type == "M": + log_type = "Mod-pkg-service" + else: + log.info("%s failed" % line) + else: + log.info("%s failed" % line) + return log_type, branch_name, proj, pkg, new_proj + +# config obs action +def config_obs_by_submit(log_type, branch_name, proj, new_proj, pkg, exist_flag): + if log_type == "Add-pkg": + if exist_flag == 0: + add_pkg(proj, pkg, branch_name) + elif log_type == "Del-pkg": + del_pkg(proj, pkg) + elif log_type == "Del-pkg-service": + del_pkg_service(proj, pkg) + elif log_type == "Mod-pkg-service": + modify_pkg_service(proj, pkg, branch_name) + elif log_type == "Mod-pkg-meta": + modify_pkg_meta(proj, pkg, branch_name) + elif log_type == "Change-pkg-prj": + change_pkg_prj(proj, new_proj, pkg, branch_name) + +# deal with relationship +def deal_import_param(): + url = "https://gitee.com/src-openeuler/obs_meta.git" + git_clone(url, obs_meta_path) + os.chdir(obs_meta_path) + os.popen("git diff --name-status HEAD~1 HEAD~0 > %s" % patch_file_path) + os.chdir(work_dir) + pattern_string = ['.meta', '.prjconf', '/_service', '/_meta'] + for pattern in pattern_string: + data = open(patch_file_path, 'r') + for line in data: + cmd1 = 'echo "%s" | grep "%s$"' % (line.strip(), pattern) + if os.popen(cmd1).read(): + cmd2 = 'echo "%s" | grep "_service" | grep "\.osc"' % line + if os.popen(cmd2).read(): + log.warning("illeagl modify,skip ...") + continue + log_type, branch_name, proj, pkg, new_prj = parse_git_log(line.strip('\n')) + log.info("log_type:%s branch_name:%s proj:%s pkg:%s new_prj:%s" %(log_type, branch_name, proj, pkg, new_prj)) + exist_flag = 0 + prj_list = ['openEuler:Mainline', 'openEuler:Factory', 'openEuler:Epol', 'openEuler:Extras', 'bringInRely'] + for p in prj_list: + cmd3 = "osc ls %s 2>&1 | grep -q -Fx %s" % (p, pkg) + if os.system(cmd3) == 0: + if p == proj: + log.info("package %s hava existed in obs project %s" % (pkg, p)) + exist_flag = 1 + log.info("start config_obs_by_submit") + config_obs_by_submit(log_type, branch_name, proj, new_prj, pkg, exist_flag) + log.info("end config_obs_by_submit") + log.info("#"*50) + else: + continue + data.close() + +if __name__ == "__main__": +# giteeUserName = sys.argv[1] +# giteeUserPwd = sys.argv[2] +# pre_env() + deal_import_param() -- Gitee From 5ace8957910fcb008249921037f759298a1e75ab Mon Sep 17 00:00:00 2001 From: wangchong1995924 <15229716099@163.com> Date: Mon, 26 Oct 2020 19:24:03 +0800 Subject: [PATCH 2/6] fix some bug --- core/obs_package_admc.py | 509 +++++++++++++++++++++------------------ 1 file changed, 273 insertions(+), 236 deletions(-) diff --git a/core/obs_package_admc.py b/core/obs_package_admc.py index dce797e..d824ae8 100755 --- a/core/obs_package_admc.py +++ b/core/obs_package_admc.py @@ -8,248 +8,285 @@ import re sys.path.append('../') from common.log_obs import log -work_dir = "/jenkins_home/workspace/obs_meta_update/openeuler_jenkins" -obs_meta_path = os.path.join(work_dir, "obs_meta") -patch_file_path = os.path.join(work_dir, "diff_patch") -giteeUserName = "openeuler-ci-bot" +class OBSManager(): + def __init__(self, giteeUserName, giteeUserPwd): + """ + obs_meta_path: obs_meta dir path + patch_file_path: diff_patch file path + """ + self.work_dir = "/jenkins_home/workspace/obs_meta_update/openeuler_jenkins" + self.obs_meta_path = os.path.join(self.work_dir, "obs_meta") + self.patch_file_path = os.path.join(self.work_dir, "diff_patch") + self.giteeUserName = giteeUserName + self.giteeUserPwd = giteeUserPwd -# initialize the workdir -def pre_env(): - if os.path.exists(work_dir): - shutil.rmtree(work_dir) - os.makedirs(work_dir) - -# git clone -def git_clone(git_url, git_path): - if os.path.exists(git_path): - shutil.rmtree(git_path) - git.Repo.clone_from(url = git_url, to_path = git_path) - -# add project package -def add_pkg(proj, pkg, branch_name): - log.info("Begin add_pkg") - os.chdir(work_dir) - if os.path.exists(proj): - shutil.rmtree(proj) - if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: - os.system("osc co %s %s &>/dev/null" %(proj, pkg)) - else: - os.system("osc co %s `osc ls %s | sed -n '1p'` &>/dev/null" %(proj, proj)) - pkg_path = os.path.join(obs_meta_path, '%s/%s/%s' % (branch_name, proj, pkg)) - os.chdir(proj) - if os.path.exists(pkg): - os.system("cp -rf %s ." % pkg_path) - cmd = "osc status | grep ^? | awk '{print 2}'" - new_file = os.popen(cmd).read() - if len(new_file): - os.system("osc add %s" %new_file) - else: - os.system("cp -rf %s ." % pkg_path) - rm_dir = os.path.join('%s/.osc' % pkg) - if os.path.exists(rm_dir): - os.system("rm -rf %s" % rm_dir) - os.system("osc add %s" % pkg) - new_file = os.popen("osc status").read() - if len(new_file): - log.info("osc ci -m 'add by %s'" % giteeUserName) -# os.system("osc ci -m 'add by %s'" % giteeUserName) - os.chdir(work_dir) - log.info("END add_pkg") - -# use by check_pkg function -def add_pkg_service(proj, pkg, branch_name): - log.info("Begin add_pkg_service") - os.chdir("%s/%s/%s"% (obs_meta_path, branch_name, proj)) - os.system("git pull &>/dev/null") - if not os.path.exists(pkg): - os.mkdir(pkg) - os.chdir(pkg) - f = open("_service",'w') - f.write('\n') - f.write(' \n') - f.write(' repo\n') - if branch_name == "master": - f.write(' next/openEuler/%s\n'% pkg) - else: - f.write(' next/%s/%s\n'% (branch_name, pkg)) - f.write(' \n') - f.write('\n') - f.close() - os.chdir("%s/%s/%s"% (obs_meta_path, branch_name, proj)) - ''' - os.popen("git add %s" % pkg) - os.popen("git commit -m 'add %s file _service by %s'"% (pkg, giteeUserName)) - os.popen("git push") - ''' - add_pkg(proj, pkg, branch_name) - log.info("END add_pkg_service") - -# delete the project package -def del_pkg(proj, pkg): - log.info("Begin del_pkg") - os.chdir(work_dir) - proj_path = os.path.join(work_dir, proj) - if os.path.exists(proj_path): - shutil.rmtree(proj) - if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: - os.system("osc co %s %s &>/dev/null" %(proj, pkg)) + def _pre_env(): + """ + initialize the workdir + """ + if os.path.exists(self.work_dir): + shutil.rmtree(self.work_dir) + os.makedirs(self.work_dir) + + def _git_clone(git_url, git_path): + """ + git clone function + """ + if os.path.exists(git_path): + shutil.rmtree(git_path) + git.Repo.clone_from(url = git_url, to_path = git_path) + + def add_pkg(proj, pkg, branch_name): + """ + add project package + """ + log.info("Begin add_pkg") + os.chdir(self.work_dir) + if os.path.exists(proj): + shutil.rmtree(proj) + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: + os.system("osc co %s %s &>/dev/null" % (proj, pkg)) + else: + os.system("osc co %s `osc ls %s | sed -n '1p'` &>/dev/null" % (proj, proj)) + pkg_path = os.path.join(self.obs_meta_path, '%s/%s/%s' % (branch_name, proj, pkg)) os.chdir(proj) - os.system("osc rm %s" % pkg) - log.info("osc ci -m 'delete by %s'" % giteeUserName) -# os.system("osc ci -m 'delete by %s'" % giteeUserName) - os.chdir(work_dir) - log.info("END del_pkg") - -# delete the service file for the package -def del_pkg_service(proj, pkg): - log.info("Begin del_pkg_service") - if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: - log.warning("%s %s not found" % (proj, pkg)) - return 0 - os.chdir(work_dir) - proj_path = os.path.join(work_dir, proj) - pkg_path = os.path.join(proj_path, pkg) - if os.path.exists(proj_path): - shutil.rmtree(proj) - os.system("osc co %s %s &>/dev/null" %(proj, pkg)) - os.chdir(pkg_path) - os.system("test -f _service && osc rm _service") - log.info("osc ci -m 'delete by %s'" % giteeUserName) -# os.system("osc ci -m 'delete by %s'" % giteeUserName) - os.chdir(work_dir) - log.info("END del_pkg_service") - -# change the service file for the package -def modify_pkg_service(proj, pkg, branch_name): - log.info("Begin modify_pkg_service") - if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: - log.info("%s %s not found" % (proj, pkg)) - os.chdir(work_dir) - proj_path = os.path.join(work_dir, proj) - pkg_path = os.path.join(proj_path, pkg) - service_file_path = os.path.join(obs_meta_path, '%s/%s/%s/_service' % (branch_name, proj, pkg)) - if os.path.exists(proj_path): - shutil.rmtree(proj) - os.system("osc co %s %s &>/dev/null" %(proj, pkg)) - os.system("cp -f %s %s" %(service_file_path, pkg_path)) - os.chdir(pkg_path) - os.system("osc add _service") - log.info("osc ci -m 'modify by %s'" % giteeUserName) -# os.system("osc ci -m 'modify by %s'" % giteeUserName) - os.chdir(work_dir) - log.info("END modify_pkg_service") - -# change the package of the meta -def modify_pkg_meta(proj, pkg, branch_name): - log.info("Begin modify_pkg_meta") - if os.system("osc ls %s %s &>/dev/null" %(proj, pkg)) == 0: - log.info("%s %s not found" % (proj, pkg)) - os.chdir(work_dir) - file_path = os.path.join(obs_meta_path, "%s/%s/%s/.osc/_meta" % (branch_name, proj, pkg)) - cmd = "osc meta pkg %s %s --file=%s | grep ^Done." % (proj, pkg, file_path) - if os.system(cmd) != 0: - log.info("%s/%s/.osc/_meta deal error" % (proj, pkg)) - log.info("END modify_pkg_meta") - -# change the package of the project -def change_pkg_prj(proj, new_proj, pkg, branch_name): - log.info("Begin change_pkg_prj") - del_pkg(proj, pkg) - add_pkg(new_proj, pkg, branch_name) - log.info("END change_pkg_prj") + if os.path.exists(pkg): + os.system("cp -rf %s ." % pkg_path) + cmd = "osc status | grep ^? | awk '{print 2}'" + new_file = os.popen(cmd).read() + if len(new_file): + os.system("osc add %s" % new_file) + else: + os.system("cp -rf %s ." % pkg_path) + rm_dir = os.path.join('%s/.osc' % pkg) + if os.path.exists(rm_dir): + os.system("rm -rf %s" % rm_dir) + os.system("osc add %s" % pkg) + new_file = os.popen("osc status").read() + if len(new_file): + log.info("osc ci -m 'add by %s'" % self.giteeUserName) + # os.system("osc ci -m 'add by %s'" % self. giteeUserName) + os.chdir(self.work_dir) + log.info("END add_pkg") + + def _add_pkg_service(proj, pkg, branch_name): + """ + use by check_pkg function + """ + log.info("Begin add_pkg_service") + os.chdir("%s/%s/%s" % (self.obs_meta_path, branch_name, proj)) + os.system("git pull &>/dev/null") + if not os.path.exists(pkg): + os.mkdir(pkg) + os.chdir(pkg) + f = open("_service", 'w') + f.write('\n') + f.write(' \n') + f.write(' repo\n') + if branch_name == "master": + f.write(' next/openEuler/%s\n' % pkg) + else: + f.write(' next/%s/%s\n' % (branch_name, pkg)) + f.write(' \n') + f.write('\n') + f.close() + os.chdir("%s/%s/%s" % (self.obs_meta_path, branch_name, proj)) + os.popen("git add %s" % pkg) + os.popen("git commit -m 'add %s file _service by %s'" % (pkg, self.giteeUserName)) + os.popen("git push") + add_pkg(proj, pkg, branch_name) + log.info("END add_pkg_service") + + def del_pkg(proj, pkg): + """ + delete the project package + """ + log.info("Begin del_pkg") + os.chdir(self.work_dir) + proj_path = os.path.join(self.work_dir, proj) + if os.path.exists(proj_path): + shutil.rmtree(proj) + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: + os.system("osc co %s %s &>/dev/null" % (proj, pkg)) + os.chdir(proj) + os.system("osc rm %s" % pkg) + log.info("osc ci -m 'delete by %s'" % self.giteeUserName) + # os.system("osc ci -m 'delete by %s'" % self.giteeUserName) + os.chdir(self.work_dir) + log.info("END del_pkg") + + def del_pkg_service(proj, pkg): + """ + delete the service file for the package + """ + log.info("Begin del_pkg_service") + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: + log.warning("%s %s not found" % (proj, pkg)) + return 0 + os.chdir(self.work_dir) + proj_path = os.path.join(self.work_dir, proj) + pkg_path = os.path.join(proj_path, pkg) + if os.path.exists(proj_path): + shutil.rmtree(proj) + os.system("osc co %s %s &>/dev/null" % (proj, pkg)) + os.chdir(pkg_path) + os.system("test -f _service && osc rm _service") + log.info("osc ci -m 'delete by %s'" % self.giteeUserName) + # os.system("osc ci -m 'delete by %s'" % self.giteeUserName) + os.chdir(self.work_dir) + log.info("END del_pkg_service") + + def modify_pkg_service(proj, pkg, branch_name): + """ + change the service file for the package + """ + log.info("Begin modify_pkg_service") + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: + log.info("%s %s not found" % (proj, pkg)) + os.chdir(self.work_dir) + proj_path = os.path.join(self.work_dir, proj) + pkg_path = os.path.join(proj_path, pkg) + service_file_path = os.path.join(self.obs_meta_path, '%s/%s/%s/_service' % (branch_name, proj, pkg)) + if os.path.exists(proj_path): + shutil.rmtree(proj) + os.system("osc co %s %s &>/dev/null" % (proj, pkg)) + os.system("cp -f %s %s" % (service_file_path, pkg_path)) + os.chdir(pkg_path) + os.system("osc add _service") + log.info("osc ci -m 'modify by %s'" % self.giteeUserName) + # os.system("osc ci -m 'modify by %s'" % self.giteeUserName) + os.chdir(self.work_dir) + log.info("END modify_pkg_service") + + def modify_pkg_meta(proj, pkg, branch_name): + """ + change the package of the meta + """ + log.info("Begin modify_pkg_meta") + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: + log.info("%s %s not found" % (proj, pkg)) + os.chdir(self.work_dir) + file_path = os.path.join(self.obs_meta_path, "%s/%s/%s/.osc/_meta" % (branch_name, proj, pkg)) + cmd = "osc meta pkg %s %s --file=%s | grep ^Done." % (proj, pkg, file_path) + if os.system(cmd) != 0: + log.info("%s/%s/.osc/_meta deal error" % (proj, pkg)) + log.info("END modify_pkg_meta") + + def change_pkg_prj(proj, new_proj, pkg, branch_name): + """ + change the package of the project + """ + log.info("Begin change_pkg_prj") + del_pkg(proj, pkg) + add_pkg(new_proj, pkg, branch_name) + log.info("END change_pkg_prj") + + def check_pkg(self): + """ + check the obs project and operate according to the src-openeuler.yaml file + """ + return -# deal diff_patch line mesg -def parse_git_log(line): - new_proj = '' - log_list = list(line.split()) - temp_log_type = log_list[0] - file_path = log_list[1] - if len(log_list) == 3: - new_file_path = list(line.split())[2] - else: - new_file_path = '' - branch_name, proj, pkg, file_name = str(file_path).split('/') - if len(new_file_path) != 0: - new_proj = new_file_path.split('/')[1] - log_type = "Change-pkg-prj" - elif file_name == "_meta": - if temp_log_type == "A" or temp_log_type == "M": - log_type = "Mod-pkg-meta" + def _parse_git_log(line): + """ + deal diff_patch line mesg + """ + new_proj = '' + log_list = list(line.split()) + temp_log_type = log_list[0] + file_path = log_list[1] + if len(log_list) == 3: + new_file_path = list(line.split())[2] else: - log.info("%s failed" % line) - elif file_name == "_service": - if temp_log_type == "A": - log_type = "Add-pkg" - elif temp_log_type == "D": - pkg_path = os.path.join(obs_meta_path, '%s/%s/%s' % (branch_name, proj, pkg)) - if os.path.exists(pkg_path): - log_type = "Del-pkg-service" + new_file_path = '' + branch_name, proj, pkg, file_name = str(file_path).split('/') + if len(new_file_path) != 0: + new_proj = new_file_path.split('/')[1] + log_type = "Change-pkg-prj" + elif file_name == "_meta": + if temp_log_type == "A" or temp_log_type == "M": + log_type = "Mod-pkg-meta" + else: + log.info("%s failed" % line) + elif file_name == "_service": + if temp_log_type == "A": + log_type = "Add-pkg" + elif temp_log_type == "D": + pkg_path = os.path.join(self.obs_meta_path, '%s/%s/%s' % (branch_name, proj, pkg)) + if os.path.exists(pkg_path): + log_type = "Del-pkg-service" + else: + log_type = "Del-pkg" + line = line.replace('_service', '').replace('\n', '') + pattern_message = line.replace(' ', '\t').replace('/', '\\/') + os.system("sed -i '/%s/d' %s" % (pattern_message, self.patch_file_path)) + elif temp_log_type == "M": + log_type = "Mod-pkg-service" else: - log_type = "Del-pkg" - line = line.replace('_service','').replace('\n', '') - pattern_message = line.replace(' ', '\t').replace('/', '\\/') - os.system("sed -i '/%s/d' %s" % (pattern_message, patch_file_path)) - elif temp_log_type == "M": - log_type = "Mod-pkg-service" + log.info("%s failed" % line) else: log.info("%s failed" % line) - else: - log.info("%s failed" % line) - return log_type, branch_name, proj, pkg, new_proj - -# config obs action -def config_obs_by_submit(log_type, branch_name, proj, new_proj, pkg, exist_flag): - if log_type == "Add-pkg": - if exist_flag == 0: - add_pkg(proj, pkg, branch_name) - elif log_type == "Del-pkg": - del_pkg(proj, pkg) - elif log_type == "Del-pkg-service": - del_pkg_service(proj, pkg) - elif log_type == "Mod-pkg-service": - modify_pkg_service(proj, pkg, branch_name) - elif log_type == "Mod-pkg-meta": - modify_pkg_meta(proj, pkg, branch_name) - elif log_type == "Change-pkg-prj": - change_pkg_prj(proj, new_proj, pkg, branch_name) - -# deal with relationship -def deal_import_param(): - url = "https://gitee.com/src-openeuler/obs_meta.git" - git_clone(url, obs_meta_path) - os.chdir(obs_meta_path) - os.popen("git diff --name-status HEAD~1 HEAD~0 > %s" % patch_file_path) - os.chdir(work_dir) - pattern_string = ['.meta', '.prjconf', '/_service', '/_meta'] - for pattern in pattern_string: - data = open(patch_file_path, 'r') - for line in data: - cmd1 = 'echo "%s" | grep "%s$"' % (line.strip(), pattern) - if os.popen(cmd1).read(): - cmd2 = 'echo "%s" | grep "_service" | grep "\.osc"' % line - if os.popen(cmd2).read(): - log.warning("illeagl modify,skip ...") + return log_type, branch_name, proj, pkg, new_proj + + def _config_obs_by_submit(log_type, branch_name, proj, new_proj, pkg, exist_flag): + """ + config obs action + """ + if log_type == "Add-pkg": + if exist_flag == 0: + add_pkg(proj, pkg, branch_name) + elif log_type == "Del-pkg": + del_pkg(proj, pkg) + elif log_type == "Del-pkg-service": + del_pkg_service(proj, pkg) + elif log_type == "Mod-pkg-service": + modify_pkg_service(proj, pkg, branch_name) + elif log_type == "Mod-pkg-meta": + modify_pkg_meta(proj, pkg, branch_name) + elif log_type == "Change-pkg-prj": + change_pkg_prj(proj, new_proj, pkg, branch_name) + + def _deal_import_param(self): + """ + deal with relationship + """ + url = "https://gitee.com/src-openeuler/obs_meta.git" + git_clone(url, self.obs_meta_path) + os.chdir(self.obs_meta_path) + os.popen("git diff --name-status HEAD~1 HEAD~0 > %s" % self.patch_file_path) + os.chdir(self.work_dir) + pattern_string = ['.meta', '.prjconf', '/_service', '/_meta'] + for pattern in pattern_string: + data = open(self.patch_file_path, 'r') + for line in data: + cmd1 = 'echo "%s" | grep "%s$"' % (line.strip(), pattern) + if os.popen(cmd1).read(): + cmd2 = 'echo "%s" | grep "_service" | grep "\.osc"' % line + if os.popen(cmd2).read(): + log.warning("illeagl modify,skip ...") + continue + log_type, branch_name, proj, pkg, new_prj = _parse_git_log(line.strip('\n')) + log.info("log_type:%s branch_name:%s proj:%s pkg:%s new_prj:%s" + % (log_type, branch_name, proj, pkg, new_prj)) + exist_flag = 0 + prj_list = ['openEuler:Mainline', 'openEuler:Factory', + 'openEuler:Epol', 'openEuler:Extras', 'bringInRely'] + for p in prj_list: + cmd3 = "osc ls %s 2>&1 | grep -q -Fx %s" % (p, pkg) + if os.system(cmd3) == 0: + if p == proj: + log.info("package %s hava existed in obs project %s" % (pkg, p)) + exist_flag = 1 + log.info("start config_obs_by_submit") + _config_obs_by_submit(log_type, branch_name, proj, new_prj, pkg, exist_flag) + log.info("end config_obs_by_submit") + else: continue - log_type, branch_name, proj, pkg, new_prj = parse_git_log(line.strip('\n')) - log.info("log_type:%s branch_name:%s proj:%s pkg:%s new_prj:%s" %(log_type, branch_name, proj, pkg, new_prj)) - exist_flag = 0 - prj_list = ['openEuler:Mainline', 'openEuler:Factory', 'openEuler:Epol', 'openEuler:Extras', 'bringInRely'] - for p in prj_list: - cmd3 = "osc ls %s 2>&1 | grep -q -Fx %s" % (p, pkg) - if os.system(cmd3) == 0: - if p == proj: - log.info("package %s hava existed in obs project %s" % (pkg, p)) - exist_flag = 1 - log.info("start config_obs_by_submit") - config_obs_by_submit(log_type, branch_name, proj, new_prj, pkg, exist_flag) - log.info("end config_obs_by_submit") - log.info("#"*50) - else: - continue - data.close() - + data.close() + if __name__ == "__main__": -# giteeUserName = sys.argv[1] -# giteeUserPwd = sys.argv[2] -# pre_env() - deal_import_param() + giteeUserName = sys.argv[1] + giteeUserPwd = sys.argv[2] + om = OBSManager(giteeUserName, giteeUserPwd) + #om.pre_env() + om._deal_import_param() -- Gitee From 36e151ad6a94463061ea57f4833e68397e963641 Mon Sep 17 00:00:00 2001 From: wangchong1995924 <15229716099@163.com> Date: Mon, 26 Oct 2020 19:46:25 +0800 Subject: [PATCH 3/6] fix bug --- core/obs_package_admc.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/core/obs_package_admc.py b/core/obs_package_admc.py index d824ae8..2a9f5f1 100755 --- a/core/obs_package_admc.py +++ b/core/obs_package_admc.py @@ -1,5 +1,8 @@ #!/bin/env python3 - +""" +created by: wangchong +date: 2020-10-26 +""" import shutil import sys import git @@ -8,7 +11,10 @@ import re sys.path.append('../') from common.log_obs import log -class OBSManager(): +class OBSManager(object): + """ + obs project package add delete modify check + """ def __init__(self, giteeUserName, giteeUserPwd): """ obs_meta_path: obs_meta dir path @@ -226,7 +232,8 @@ class OBSManager(): log.info("%s failed" % line) else: log.info("%s failed" % line) - return log_type, branch_name, proj, pkg, new_proj + mesg_list = [ log_type, branch_name, proj, pkg, new_proj ] + return mesg_list def _config_obs_by_submit(log_type, branch_name, proj, new_proj, pkg, exist_flag): """ -- Gitee From ba48eb95c65931be4cbf80e606fe3a5f8378d067 Mon Sep 17 00:00:00 2001 From: wangchong1995924 <15229716099@163.com> Date: Mon, 26 Oct 2020 19:47:15 +0800 Subject: [PATCH 4/6] fix bug --- core/obs_package_admc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/obs_package_admc.py b/core/obs_package_admc.py index 2a9f5f1..fce235b 100755 --- a/core/obs_package_admc.py +++ b/core/obs_package_admc.py @@ -232,7 +232,7 @@ class OBSManager(object): log.info("%s failed" % line) else: log.info("%s failed" % line) - mesg_list = [ log_type, branch_name, proj, pkg, new_proj ] + mesg_list = [log_type, branch_name, proj, pkg, new_proj] return mesg_list def _config_obs_by_submit(log_type, branch_name, proj, new_proj, pkg, exist_flag): -- Gitee From 4c2e8f1bb9401628c06177abd548a02e6c0e0f36 Mon Sep 17 00:00:00 2001 From: wangchong1995924 <15229716099@163.com> Date: Tue, 27 Oct 2020 09:27:16 +0800 Subject: [PATCH 5/6] add obs_package_admc.py version 2 --- core/obs_package_admc.py | 105 +++++++++++++++------------------------ 1 file changed, 41 insertions(+), 64 deletions(-) diff --git a/core/obs_package_admc.py b/core/obs_package_admc.py index fce235b..95e97b6 100755 --- a/core/obs_package_admc.py +++ b/core/obs_package_admc.py @@ -3,11 +3,11 @@ created by: wangchong date: 2020-10-26 """ -import shutil -import sys -import git import os import re +import sys +import git +import shutil sys.path.append('../') from common.log_obs import log @@ -26,7 +26,7 @@ class OBSManager(object): self.giteeUserName = giteeUserName self.giteeUserPwd = giteeUserPwd - def _pre_env(): + def _pre_env(self): """ initialize the workdir """ @@ -34,7 +34,7 @@ class OBSManager(object): shutil.rmtree(self.work_dir) os.makedirs(self.work_dir) - def _git_clone(git_url, git_path): + def _git_clone(self, git_url, git_path): """ git clone function """ @@ -42,11 +42,10 @@ class OBSManager(object): shutil.rmtree(git_path) git.Repo.clone_from(url = git_url, to_path = git_path) - def add_pkg(proj, pkg, branch_name): + def add_pkg(self, proj, pkg, branch_name): """ add project package """ - log.info("Begin add_pkg") os.chdir(self.work_dir) if os.path.exists(proj): shutil.rmtree(proj) @@ -70,16 +69,13 @@ class OBSManager(object): os.system("osc add %s" % pkg) new_file = os.popen("osc status").read() if len(new_file): - log.info("osc ci -m 'add by %s'" % self.giteeUserName) - # os.system("osc ci -m 'add by %s'" % self. giteeUserName) + os.system("osc ci -m 'add by %s'" % self.giteeUserName) os.chdir(self.work_dir) - log.info("END add_pkg") - def _add_pkg_service(proj, pkg, branch_name): + def _add_pkg_service(self, proj, pkg, branch_name): """ use by check_pkg function """ - log.info("Begin add_pkg_service") os.chdir("%s/%s/%s" % (self.obs_meta_path, branch_name, proj)) os.system("git pull &>/dev/null") if not os.path.exists(pkg): @@ -97,17 +93,15 @@ class OBSManager(object): f.write('\n') f.close() os.chdir("%s/%s/%s" % (self.obs_meta_path, branch_name, proj)) - os.popen("git add %s" % pkg) - os.popen("git commit -m 'add %s file _service by %s'" % (pkg, self.giteeUserName)) - os.popen("git push") + os.system("git add %s" % pkg) + os.system("git commit -m 'add %s _service file by %s'" % (pkg, self.giteeUserName)) + os.system("git push") add_pkg(proj, pkg, branch_name) - log.info("END add_pkg_service") - def del_pkg(proj, pkg): + def del_pkg(self, proj, pkg): """ delete the project package """ - log.info("Begin del_pkg") os.chdir(self.work_dir) proj_path = os.path.join(self.work_dir, proj) if os.path.exists(proj_path): @@ -116,16 +110,13 @@ class OBSManager(object): os.system("osc co %s %s &>/dev/null" % (proj, pkg)) os.chdir(proj) os.system("osc rm %s" % pkg) - log.info("osc ci -m 'delete by %s'" % self.giteeUserName) - # os.system("osc ci -m 'delete by %s'" % self.giteeUserName) + os.system("osc ci -m 'delete by %s'" % self.giteeUserName) os.chdir(self.work_dir) - log.info("END del_pkg") - def del_pkg_service(proj, pkg): + def del_pkg_service(self, proj, pkg): """ delete the service file for the package """ - log.info("Begin del_pkg_service") if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: log.warning("%s %s not found" % (proj, pkg)) return 0 @@ -137,18 +128,15 @@ class OBSManager(object): os.system("osc co %s %s &>/dev/null" % (proj, pkg)) os.chdir(pkg_path) os.system("test -f _service && osc rm _service") - log.info("osc ci -m 'delete by %s'" % self.giteeUserName) - # os.system("osc ci -m 'delete by %s'" % self.giteeUserName) + os.system("osc ci -m 'delete by %s'" % self.giteeUserName) os.chdir(self.work_dir) - log.info("END del_pkg_service") - def modify_pkg_service(proj, pkg, branch_name): + def modify_pkg_service(self, proj, pkg, branch_name): """ change the service file for the package """ - log.info("Begin modify_pkg_service") if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: - log.info("%s %s not found" % (proj, pkg)) + log.warning("%s %s not found" % (proj, pkg)) os.chdir(self.work_dir) proj_path = os.path.join(self.work_dir, proj) pkg_path = os.path.join(proj_path, pkg) @@ -159,33 +147,27 @@ class OBSManager(object): os.system("cp -f %s %s" % (service_file_path, pkg_path)) os.chdir(pkg_path) os.system("osc add _service") - log.info("osc ci -m 'modify by %s'" % self.giteeUserName) - # os.system("osc ci -m 'modify by %s'" % self.giteeUserName) + os.system("osc ci -m 'modify by %s'" % self.giteeUserName) os.chdir(self.work_dir) - log.info("END modify_pkg_service") - def modify_pkg_meta(proj, pkg, branch_name): + def modify_pkg_meta(self, proj, pkg, branch_name): """ change the package of the meta """ - log.info("Begin modify_pkg_meta") if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: - log.info("%s %s not found" % (proj, pkg)) + log.warning("%s %s not found" % (proj, pkg)) os.chdir(self.work_dir) file_path = os.path.join(self.obs_meta_path, "%s/%s/%s/.osc/_meta" % (branch_name, proj, pkg)) cmd = "osc meta pkg %s %s --file=%s | grep ^Done." % (proj, pkg, file_path) if os.system(cmd) != 0: - log.info("%s/%s/.osc/_meta deal error" % (proj, pkg)) - log.info("END modify_pkg_meta") + log.error("%s/%s/.osc/_meta deal error" % (proj, pkg)) - def change_pkg_prj(proj, new_proj, pkg, branch_name): + def change_pkg_prj(self, proj, new_proj, pkg, branch_name): """ change the package of the project """ - log.info("Begin change_pkg_prj") del_pkg(proj, pkg) add_pkg(new_proj, pkg, branch_name) - log.info("END change_pkg_prj") def check_pkg(self): """ @@ -193,18 +175,17 @@ class OBSManager(object): """ return - def _parse_git_log(line): + def _parse_git_log(self, line): """ deal diff_patch line mesg """ new_proj = '' + new_file_path = '' log_list = list(line.split()) temp_log_type = log_list[0] file_path = log_list[1] if len(log_list) == 3: new_file_path = list(line.split())[2] - else: - new_file_path = '' branch_name, proj, pkg, file_name = str(file_path).split('/') if len(new_file_path) != 0: new_proj = new_file_path.split('/')[1] @@ -213,7 +194,7 @@ class OBSManager(object): if temp_log_type == "A" or temp_log_type == "M": log_type = "Mod-pkg-meta" else: - log.info("%s failed" % line) + log.error("%s failed" % line) elif file_name == "_service": if temp_log_type == "A": log_type = "Add-pkg" @@ -229,38 +210,38 @@ class OBSManager(object): elif temp_log_type == "M": log_type = "Mod-pkg-service" else: - log.info("%s failed" % line) + log.error("%s failed" % line) else: - log.info("%s failed" % line) + log.error("%s failed" % line) mesg_list = [log_type, branch_name, proj, pkg, new_proj] return mesg_list - def _config_obs_by_submit(log_type, branch_name, proj, new_proj, pkg, exist_flag): + def _config_obs_by_submit(self, log_type, branch_name, proj, new_proj, pkg, exist_flag): """ config obs action """ if log_type == "Add-pkg": if exist_flag == 0: - add_pkg(proj, pkg, branch_name) + self.add_pkg(proj, pkg, branch_name) elif log_type == "Del-pkg": - del_pkg(proj, pkg) + self.del_pkg(proj, pkg) elif log_type == "Del-pkg-service": - del_pkg_service(proj, pkg) + self.del_pkg_service(proj, pkg) elif log_type == "Mod-pkg-service": - modify_pkg_service(proj, pkg, branch_name) + self.modify_pkg_service(proj, pkg, branch_name) elif log_type == "Mod-pkg-meta": - modify_pkg_meta(proj, pkg, branch_name) + self.modify_pkg_meta(proj, pkg, branch_name) elif log_type == "Change-pkg-prj": - change_pkg_prj(proj, new_proj, pkg, branch_name) + self.change_pkg_prj(proj, new_proj, pkg, branch_name) def _deal_import_param(self): """ deal with relationship """ - url = "https://gitee.com/src-openeuler/obs_meta.git" - git_clone(url, self.obs_meta_path) + url = "https://%s:%s@gitee.com/src-openeuler/obs_meta.git" % (self.giteeUserName, self.giteeUserPwd) + self._git_clone(url, self.obs_meta_path) os.chdir(self.obs_meta_path) - os.popen("git diff --name-status HEAD~1 HEAD~0 > %s" % self.patch_file_path) + os.system("git diff --name-status HEAD~1 HEAD~0 > %s" % self.patch_file_path) os.chdir(self.work_dir) pattern_string = ['.meta', '.prjconf', '/_service', '/_meta'] for pattern in pattern_string: @@ -272,7 +253,7 @@ class OBSManager(object): if os.popen(cmd2).read(): log.warning("illeagl modify,skip ...") continue - log_type, branch_name, proj, pkg, new_prj = _parse_git_log(line.strip('\n')) + log_type, branch_name, proj, pkg, new_prj = self._parse_git_log(line.strip('\n')) log.info("log_type:%s branch_name:%s proj:%s pkg:%s new_prj:%s" % (log_type, branch_name, proj, pkg, new_prj)) exist_flag = 0 @@ -284,16 +265,12 @@ class OBSManager(object): if p == proj: log.info("package %s hava existed in obs project %s" % (pkg, p)) exist_flag = 1 - log.info("start config_obs_by_submit") - _config_obs_by_submit(log_type, branch_name, proj, new_prj, pkg, exist_flag) - log.info("end config_obs_by_submit") + self._config_obs_by_submit(log_type, branch_name, proj, new_prj, pkg, exist_flag) else: continue data.close() if __name__ == "__main__": - giteeUserName = sys.argv[1] - giteeUserPwd = sys.argv[2] - om = OBSManager(giteeUserName, giteeUserPwd) - #om.pre_env() + om = OBSManager(sys.argv[1], sys.argv[2]) + om._pre_env() om._deal_import_param() -- Gitee From e5d7d3fa79991b82e09569b52fc88693ee2d9678 Mon Sep 17 00:00:00 2001 From: wangchong1995924 <15229716099@163.com> Date: Thu, 29 Oct 2020 10:11:42 +0800 Subject: [PATCH 6/6] add obs_package_admc.py version 3 --- core/obs_package_admc.py | 245 +++++++++++++++++++++++++++++---------- 1 file changed, 182 insertions(+), 63 deletions(-) diff --git a/core/obs_package_admc.py b/core/obs_package_admc.py index 95e97b6..b2365de 100755 --- a/core/obs_package_admc.py +++ b/core/obs_package_admc.py @@ -1,8 +1,10 @@ #!/bin/env python3 + """ created by: wangchong date: 2020-10-26 """ + import os import re import sys @@ -25,6 +27,7 @@ class OBSManager(object): self.patch_file_path = os.path.join(self.work_dir, "diff_patch") self.giteeUserName = giteeUserName self.giteeUserPwd = giteeUserPwd + self.import_list = [[]] * 2000 def _pre_env(self): """ @@ -33,16 +36,31 @@ class OBSManager(object): if os.path.exists(self.work_dir): shutil.rmtree(self.work_dir) os.makedirs(self.work_dir) + os.chdir(self.work_dir) - def _git_clone(self, git_url, git_path): + def _git_clone(self, git_house): """ git clone function """ - if os.path.exists(git_path): - shutil.rmtree(git_path) - git.Repo.clone_from(url = git_url, to_path = git_path) + if git_house == "obs_meta": + if os.path.exists(self.obs_meta_path): + shutil.rmtree(self.obs_meta_path) + git_url = "https://%s:%s@gitee.com/src-openeuler/obs_meta.git" % ( + self.giteeUserName, self.giteeUserPwd) + git.Repo.clone_from(url = git_url, to_path = self.obs_meta_path) + os.chdir(self.obs_meta_path) + os.system("git diff --name-status HEAD~1 HEAD~0 > %s" + % self.patch_file_path) + if git_house == "community": + community_path = os.path.join(self.work_dir, "community") + if os.path.exists(community_path): + shutil.rmtree(community_path) + git_url = "https://%s:%s@gitee.com/openeuler/community.git" % ( + self.giteeUserName, self.giteeUserPwd) + git.Repo.clone_from(url = git_url, to_path = community_path) + os.chdir(self.work_dir) - def add_pkg(self, proj, pkg, branch_name): + def _add_pkg(self, proj, pkg, branch_name): """ add project package """ @@ -74,10 +92,13 @@ class OBSManager(object): def _add_pkg_service(self, proj, pkg, branch_name): """ - use by check_pkg function + write and push _service file in obs_meta """ - os.chdir("%s/%s/%s" % (self.obs_meta_path, branch_name, proj)) - os.system("git pull &>/dev/null") + proj_path = os.path.join(self.obs_meta_path, branch_name, proj) + if not os.path.exists(proj_path): + log.warning("obs_meta do not have %s %s" % (branch_name, proj)) + return -1 + os.chdir(proj_path) if not os.path.exists(pkg): os.mkdir(pkg) os.chdir(pkg) @@ -96,9 +117,8 @@ class OBSManager(object): os.system("git add %s" % pkg) os.system("git commit -m 'add %s _service file by %s'" % (pkg, self.giteeUserName)) os.system("git push") - add_pkg(proj, pkg, branch_name) - def del_pkg(self, proj, pkg): + def _del_pkg(self, proj, pkg): """ delete the project package """ @@ -111,15 +131,18 @@ class OBSManager(object): os.chdir(proj) os.system("osc rm %s" % pkg) os.system("osc ci -m 'delete by %s'" % self.giteeUserName) + else: + log.warning("obs %s %s not found" % (proj, pkg)) + return -1 os.chdir(self.work_dir) - def del_pkg_service(self, proj, pkg): + def _del_obs_pkg_service(self, proj, pkg): """ - delete the service file for the package + delete the obs project package service file """ - if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: - log.warning("%s %s not found" % (proj, pkg)) - return 0 + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) != 0: + log.warning("obs %s %s not found" % (proj, pkg)) + return -1 os.chdir(self.work_dir) proj_path = os.path.join(self.work_dir, proj) pkg_path = os.path.join(proj_path, pkg) @@ -131,12 +154,29 @@ class OBSManager(object): os.system("osc ci -m 'delete by %s'" % self.giteeUserName) os.chdir(self.work_dir) - def modify_pkg_service(self, proj, pkg, branch_name): + def _del_meta_pkg_service(self, branch, proj, pkg): + """ + delete the obs_meta project pkg service file + """ + if not os.path.exists(os.path.join(self.obs_meta_path, branch, proj, pkg)): + log.warning("obs_meta %s %s %s not found" % (branch, proj, pkg)) + return -1 + proj_path = os.path.join(self.obs_meta_path, branch, proj) + pkg_path = os.path.join(proj_path, pkg) + os.chdir(proj_path) + if os.path.exists(pkg_path): + os.system("rm -rf %s" % pkg) + os.system("git add -A && git commit -m 'delete by %s'" % self.giteeUserName) + os.system("git push") + os.chdir(self.work_dir) + + def _modify_pkg_service(self, proj, pkg, branch_name): """ change the service file for the package """ - if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) != 0: log.warning("%s %s not found" % (proj, pkg)) + return -1 os.chdir(self.work_dir) proj_path = os.path.join(self.work_dir, proj) pkg_path = os.path.join(proj_path, pkg) @@ -150,31 +190,26 @@ class OBSManager(object): os.system("osc ci -m 'modify by %s'" % self.giteeUserName) os.chdir(self.work_dir) - def modify_pkg_meta(self, proj, pkg, branch_name): + def _modify_pkg_meta(self, proj, pkg, branch_name): """ change the package of the meta """ - if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0: + if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) != 0: log.warning("%s %s not found" % (proj, pkg)) + return -1 os.chdir(self.work_dir) file_path = os.path.join(self.obs_meta_path, "%s/%s/%s/.osc/_meta" % (branch_name, proj, pkg)) cmd = "osc meta pkg %s %s --file=%s | grep ^Done." % (proj, pkg, file_path) if os.system(cmd) != 0: log.error("%s/%s/.osc/_meta deal error" % (proj, pkg)) - def change_pkg_prj(self, proj, new_proj, pkg, branch_name): + def _change_pkg_prj(self, proj, new_proj, pkg, branch_name): """ change the package of the project """ - del_pkg(proj, pkg) - add_pkg(new_proj, pkg, branch_name) + self._del_pkg(proj, pkg) + self._add_pkg(new_proj, pkg, branch_name) - def check_pkg(self): - """ - check the obs project and operate according to the src-openeuler.yaml file - """ - return - def _parse_git_log(self, line): """ deal diff_patch line mesg @@ -216,46 +251,18 @@ class OBSManager(object): mesg_list = [log_type, branch_name, proj, pkg, new_proj] return mesg_list - def _config_obs_by_submit(self, log_type, branch_name, proj, new_proj, pkg, exist_flag): - """ - config obs action + def _deal_some_param(self): """ - if log_type == "Add-pkg": - if exist_flag == 0: - self.add_pkg(proj, pkg, branch_name) - elif log_type == "Del-pkg": - self.del_pkg(proj, pkg) - elif log_type == "Del-pkg-service": - self.del_pkg_service(proj, pkg) - elif log_type == "Mod-pkg-service": - self.modify_pkg_service(proj, pkg, branch_name) - elif log_type == "Mod-pkg-meta": - self.modify_pkg_meta(proj, pkg, branch_name) - elif log_type == "Change-pkg-prj": - self.change_pkg_prj(proj, new_proj, pkg, branch_name) - - def _deal_import_param(self): + deal with some data and relation """ - deal with relationship - """ - url = "https://%s:%s@gitee.com/src-openeuler/obs_meta.git" % (self.giteeUserName, self.giteeUserPwd) - self._git_clone(url, self.obs_meta_path) - os.chdir(self.obs_meta_path) - os.system("git diff --name-status HEAD~1 HEAD~0 > %s" % self.patch_file_path) - os.chdir(self.work_dir) pattern_string = ['.meta', '.prjconf', '/_service', '/_meta'] + num = 0 for pattern in pattern_string: data = open(self.patch_file_path, 'r') for line in data: cmd1 = 'echo "%s" | grep "%s$"' % (line.strip(), pattern) if os.popen(cmd1).read(): - cmd2 = 'echo "%s" | grep "_service" | grep "\.osc"' % line - if os.popen(cmd2).read(): - log.warning("illeagl modify,skip ...") - continue log_type, branch_name, proj, pkg, new_prj = self._parse_git_log(line.strip('\n')) - log.info("log_type:%s branch_name:%s proj:%s pkg:%s new_prj:%s" - % (log_type, branch_name, proj, pkg, new_prj)) exist_flag = 0 prj_list = ['openEuler:Mainline', 'openEuler:Factory', 'openEuler:Epol', 'openEuler:Extras', 'bringInRely'] @@ -265,12 +272,124 @@ class OBSManager(object): if p == proj: log.info("package %s hava existed in obs project %s" % (pkg, p)) exist_flag = 1 - self._config_obs_by_submit(log_type, branch_name, proj, new_prj, pkg, exist_flag) + self.import_list[num] = [log_type, branch_name, proj, pkg, new_prj, exist_flag] + num += 1 else: continue data.close() - + + def obs_pkg_admc(self): + """ + obs project package add, delete, modify, check + """ + self._pre_env() + self._git_clone("obs_meta") + self._deal_some_param() + for i in range(len(self.import_list)): + if len(self.import_list[i]): + log_type, branch_name, proj, pkg, new_proj, exist_flag = self.import_list[i] + log.info("log_type:%s branch_name:%s proj:%s pkg:%s new_proj:%s exist_flag:%s" + % (log_type, branch_name, proj, pkg, new_proj, exist_flag)) + if log_type == "Add-pkg": + if exist_flag == 0: + self._add_pkg(proj, pkg, branch_name) + elif log_type == "Del-pkg": + self._del_pkg(proj, pkg) + elif log_type == "Del-pkg-service": + self._del_obs_pkg_service(proj, pkg) + elif log_type == "Mod-pkg-service": + self._modify_pkg_service(proj, pkg, branch_name) + elif log_type == "Mod-pkg-meta": + self._modify_pkg_meta(proj, pkg, branch_name) + elif log_type == "Change-pkg-prj": + self._change_pkg_prj(proj, new_proj, pkg, branch_name) + else: + return 0 + + def check_obs_pkg(self): + """ + check the obs project and operate according to the src-openeuler.yaml file + """ + self._pre_env() + self._git_clone("community") + self._git_clone("obs_meta") + os.chdir(self.work_dir) + yaml_path = os.path.join(self.work_dir, "community/repository") + f1 = open("%s/src-openeuler.yaml" % yaml_path, 'r') + yaml_dict = {} + meta_bp_dict = {} + pkg_branch_dict = {} + branch_proj_dict = {'master':'openEuler:Factory', + 'openEuler-20.09':'openEuler:20.09', + 'openEuler-20.03-LTS':'openEuler:20.03:LTS', + 'openEuler-20.03-LTS-Next':'openEuler:20.03:LTS:Next', + } + for line in f1: + if re.search("- name:", line): + pkg_name = line.split()[2] + if re.search(" - ", line): + branch = line.split()[1] + if branch != "openEuler1.0-base" and branch != "openEuler1.0": + yaml_dict.setdefault(pkg_name,[]).append(branch) + f1.close() + os.chdir(self.obs_meta_path) + cmd = "find | grep _service | grep -Ev 'RISC-V|OBS_PRJ_meta|openEuler-EPOL-LTS|openEuler-20.03-SP1' | \ + awk -F '/' '{print $2,$3,$(NF-1)}' | sort | uniq > %s/res.txt" % self.work_dir + while os.system(cmd) != 0: + continue + os.chdir(self.work_dir) + f2 = open("%s/res.txt" % self.work_dir, 'r') + for line in f2: + br = line.strip().split()[0] + proj = line.strip().split()[1] + name = line.strip().split()[2] + meta_bp_dict.setdefault(br,[]).append(proj) + pkg_branch_dict.setdefault(name,[]).append(br) + f2.close() + log.info("check BEGIN") + log.info("check stage 1:") + for pkg, branch in yaml_dict.items(): + if pkg in pkg_branch_dict: + branch.sort() + pkg_branch_dict[pkg].sort() + if branch == pkg_branch_dict[pkg]: + continue + else: + diff_add_br = set(branch).difference(set(pkg_branch_dict[pkg])) + for diff in diff_add_br: + self._add_pkg_service(branch_proj_dict[diff], pkg, diff) + self._add_pkg(branch_proj_dict[diff], pkg, diff) + else: + for need_add_br in yaml_dict[pkg]: + self._add_pkg_service(branch_proj_dict[need_add_br], pkg, need_add_br) + self._add_pkg(branch_proj_dict[need_add_br], pkg, need_add_br) + log.info("check stage 2:") + for pkg, branch in pkg_branch_dict.items(): + if pkg in yaml_dict: + branch.sort() + yaml_dict[pkg].sort() + if branch == yaml_dict[pkg]: + continue + else: + diff_del_br = set(branch).difference(set(yaml_dict[pkg])) + for diff in diff_del_br: + for proj in meta_bp_dict[diff]: + self._del_meta_pkg_service(diff, proj, pkg) + self._del_pkg(proj, pkg) + else: + for need_del_br in pkg_branch_dict[pkg]: + for proj in meta_bp_dict[need_del_br]: + self._del_meta_pkg_service(need_del_br, proj, pkg) + self._del_pkg(proj, pkg) + os.remove("%s/res.txt" % self.work_dir) + log.info("check END") + + if __name__ == "__main__": + """ + sys.argv[1]: git user name + sys.argv[2]: git user pwd + """ om = OBSManager(sys.argv[1], sys.argv[2]) - om._pre_env() - om._deal_import_param() + om.obs_pkg_admc() + om.check_obs_pkg() -- Gitee