diff --git a/core/package_manager.py b/core/package_manager.py
index be0d13e64a1c65a6aa64d7fac18d6daa91aa38fc..14454209dce71c853def04b3eab78ecbe16bcf7d 100755
--- a/core/package_manager.py
+++ b/core/package_manager.py
@@ -24,6 +24,7 @@ import sys
import yaml
import shutil
import configparser
+from concurrent.futures import ThreadPoolExecutor
from core.gitee_to_obs import SYNCCode
current_path = os.path.join(os.path.split(os.path.realpath(__file__))[0])
sys.path.append(os.path.join(current_path, ".."))
@@ -47,8 +48,7 @@ class OBSPkgManager(object):
self.giteeUserName = self.kwargs["gitee_user"]
self.giteeUserPwd = self.kwargs["gitee_pwd"]
self.sync_code = self.kwargs["sync_code"]
- self.import_list = []
- self.parent_dir = ''
+ self.multi_version_dir = ''
def _pre_env(self):
"""
@@ -58,7 +58,7 @@ class OBSPkgManager(object):
shutil.rmtree(self.work_dir)
os.makedirs(self.work_dir)
os.chdir(self.work_dir)
-
+
def _git_clone(self, git_house):
"""
git clone function
@@ -76,61 +76,51 @@ class OBSPkgManager(object):
ret = os.popen("git lfs --depth 1 clone %s" % git_url).read()
log.debug(ret)
os.chdir(self.work_dir)
-
+
def _add_pkg(self, proj, pkg, branch_name):
"""
add project package
"""
- os.chdir(self.work_dir)
- if os.path.exists(proj):
- shutil.rmtree(proj)
- if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0:
- if os.system("osc co %s %s &>/dev/null" % (proj, pkg)) == 0:
- log.info("osc co %s %s success!" % (proj, pkg))
- else:
- log.info("osc co %s %s failed!" % (proj, pkg))
- else:
- if os.system("osc co %s `osc ls %s | sed -n '1p'` &>/dev/null" % (proj, proj)) == 0:
- log.info("osc co %s success!" % proj)
- else:
- log.info("osc co %s failed!" % proj)
- pkg_path = os.path.join(self.obs_meta_path, self.parent_dir, '%s/%s/%s' % (branch_name, proj, pkg))
- if os.path.exists(proj):
- os.chdir(proj)
- else:
- log.info("add %s %s failed!" % (proj, pkg))
+ pkg_path = os.path.join(self.obs_meta_path, self.multi_version_dir, branch_name, proj, pkg)
+ _tmpdir = os.popen("mktemp -d").read().strip('\n')
+ cmd = "cd %s && osc co %s `osc ls %s 2>/dev/null | sed -n '1p'` &>/dev/null && cd -" % (_tmpdir, proj, proj)
+ ret = os.popen(cmd).read()
+ proj_tmpdir = os.path.join(_tmpdir, proj)
+ if not os.path.exists(proj_tmpdir):
+ log.error("failed to exec cmd: %s" % (cmd))
+ shutil.rmtree(_tmpdir)
return -1
- if os.path.exists(pkg):
- os.system("cp -rf %s ." % pkg_path)
- cmd = "osc status | grep ^? | awk '{print 2}'"
+ pkg_tmpdir = os.path.join(proj_tmpdir, pkg)
+ if os.path.exists(pkg_tmpdir):
+ ret = os.system("cp -rf %s %s" % (pkg_path, proj_tmpdir))
+ cmd = "cd %s && osc status | grep ^? | awk '{print 2}' && cd -" % proj_tmpdir
new_file = os.popen(cmd).read()
if len(new_file):
- os.system("osc add %s" % new_file)
+ ret = os.system("cd %s && osc add %s && cd -" % (proj_tmpdir, new_file))
else:
- os.system("cp -rf %s ." % pkg_path)
- rm_dir = os.path.join('%s/.osc' % pkg)
- if os.path.exists(rm_dir):
- os.system("rm -rf %s" % rm_dir)
- os.system("osc add %s" % pkg)
- new_file = os.popen("osc status").read()
+ rm_dir = os.path.join(pkg_tmpdir, ".osc")
+ cmd = "cp -rf %s %s && rm -rf %s && cd %s && osc add %s && cd -" % (pkg_path,
+ proj_tmpdir, rm_dir, proj_tmpdir, pkg)
+ ret = os.popen(cmd).read()
+ new_file = os.popen("cd %s && osc status && cd -" % (proj_tmpdir)).read()
if len(new_file):
- os.system("osc ci -m 'add %s by %s'" % (pkg, self.giteeUserName))
- log.info("add %s %s by %s" % (proj, pkg, self.giteeUserName))
- os.chdir(self.work_dir)
-
+ ret = os.system("cd %s && osc ci -m 'add %s by %s' && cd -" % (proj_tmpdir, pkg, self.giteeUserName))
+ log.info("add %s %s by %s succeessful" % (proj, pkg, self.giteeUserName))
+ shutil.rmtree(_tmpdir)
+
def _add_pkg_service(self, proj, pkg, branch_name):
"""
write and push _service file in obs_meta
return 0 or -1
"""
- proj_path = os.path.join(self.obs_meta_path, self.parent_dir, branch_name, proj)
+ proj_path = os.path.join(self.obs_meta_path, self.multi_version_dir, branch_name, proj)
service_file = os.path.join(proj_path, pkg, "_service")
if not os.path.exists(proj_path):
- log.warning("obs_meta do not have %s %s %s" % (self.parent_dir, branch_name, proj))
+ log.warning("obs_meta do not have %s %s %s" % (self.multi_version_dir, branch_name, proj))
return -1
if os.system("test -f %s" % service_file) == 0:
log.warning("obs_meta haved %s %s %s %s _service file, no need to add."
- % (self.parent_dir, branch_name, proj, pkg))
+ % (self.multi_version_dir, branch_name, proj, pkg))
return -1
os.chdir(proj_path)
if not os.path.exists(pkg):
@@ -142,14 +132,14 @@ class OBSPkgManager(object):
f.write(' repo\n')
if branch_name == "master":
f.write(' next/openEuler/%s\n' % pkg)
- if self.parent_dir == "":
+ if self.multi_version_dir == "":
f.write(' next/%s/%s\n' % (branch_name, pkg))
else:
- f.write(' next/%s/%s/%s\n' % (self.parent_dir, branch_name, pkg))
+ f.write(' next/%s/%s/%s\n' % (self.multi_version_dir, branch_name, pkg))
f.write(' \n')
f.write('\n')
f.close()
- os.chdir("%s/%s/%s/%s" % (self.obs_meta_path, self.parent_dir, branch_name, proj))
+ os.chdir("%s/%s/%s/%s" % (self.obs_meta_path, self.multi_version_dir, branch_name, proj))
os.system("git add %s" % pkg)
os.system("git commit -m 'add _service file by %s'" % self.giteeUserName)
log.info("add %s %s _service file by %s" % (proj, pkg, self.giteeUserName))
@@ -157,118 +147,96 @@ class OBSPkgManager(object):
if os.system("git push") == 0:
break
return 0
-
+
def _del_pkg(self, proj, pkg):
"""
delete the project package
return 0 or -1
"""
- os.chdir(self.work_dir)
- proj_path = os.path.join(self.work_dir, proj)
- if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) == 0:
- if os.path.exists(proj_path):
- shutil.rmtree(proj)
- if os.system("osc co %s %s &>/dev/null" % (proj, pkg)) == 0:
- os.chdir(proj)
- os.system("osc rm %s" % pkg)
- os.system("osc ci -m 'delete by %s'" % self.giteeUserName)
- log.info("delete %s %s by %s" % (proj, pkg, self.giteeUserName))
- else:
- log.info("delete %s %s failed!" % (proj, pkg))
- return -1
+ cmd = "osc api -X DELETE /source/%s/%s" % (proj, pkg)
+ ret = os.popen(cmd).read()
+ if "Ok" in ret:
+ log.info("delete %s %s success!" % (proj, pkg))
+ return 0
else:
- log.warning("obs %s %s not found" % (proj, pkg))
+ log.error("delete %s %s failed!" % (proj, pkg))
return -1
- os.chdir(self.work_dir)
- return 0
-
+
def _del_obs_pkg_service(self, proj, pkg):
"""
delete the obs project package service file
return 0 or -1
"""
- if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) != 0:
- log.warning("obs %s %s not found" % (proj, pkg))
- return -1
- os.chdir(self.work_dir)
- proj_path = os.path.join(self.work_dir, proj)
- pkg_path = os.path.join(proj_path, pkg)
- if os.path.exists(proj_path):
- shutil.rmtree(proj)
- if os.system("osc co %s %s &>/dev/null" % (proj, pkg)) == 0:
- os.chdir(pkg_path)
- os.system("test -f _service && osc rm _service")
- os.system("osc ci -m 'delete _service by %s'" % self.giteeUserName)
- log.info("delete obs %s %s _service by %s" % (proj, pkg, self.giteeUserName))
+ cmd = "osc api -X DELETE /source/%s/%s/_service" % (proj, pkg)
+ ret = os.popen(cmd).read()
+ if "Ok" in ret:
+ log.info("delete %s %s _service by %s successful!" % (proj, pkg, self.giteeUserName))
+ return 0
else:
- log.info("delete %s %s _service failed!" % (proj, pkg))
+ log.error("delete %s %s _service failed!" % (proj, pkg))
return -1
- os.chdir(self.work_dir)
- return 0
-
+
def _del_meta_pkg_service(self, branch, proj, pkg):
"""
delete the obs_meta project pkg service file
return 0 or -1
"""
- proj_path = os.path.join(self.obs_meta_path, self.parent_dir, branch, proj)
+ proj_path = os.path.join(self.obs_meta_path, self.multi_version_dir, branch, proj)
service_file = os.path.join(proj_path, pkg, "_service")
if os.system("test -f %s" % service_file) != 0:
- log.warning("obs_meta not have %s %s %s %s _service file" % (self.parent_dir, branch, proj, pkg))
+ log.warning("obs_meta not have %s %s %s %s _service file" % (self.multi_version_dir, branch, proj, pkg))
return -1
os.chdir(proj_path)
os.system("rm -rf %s" % pkg)
os.system("git add -A && git commit -m 'delete %s by %s'" % (pkg, self.giteeUserName))
- log.info("delete obs_meta %s %s %s %s by %s" % (self.parent_dir, branch, proj, pkg, self.giteeUserName))
+ log.info("delete obs_meta %s %s %s %s by %s" % (self.multi_version_dir, branch, proj, pkg, self.giteeUserName))
for i in range(5):
if os.system("git push") == 0:
break
os.chdir(self.work_dir)
return 0
-
+
def _modify_pkg_service(self, proj, pkg, branch_name):
"""
change the service file for the package
return 0 or -1
"""
- if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) != 0:
- log.warning("%s %s not found" % (proj, pkg))
- return -1
- os.chdir(self.work_dir)
- proj_path = os.path.join(self.work_dir, proj)
- pkg_path = os.path.join(proj_path, pkg)
- service_file_path = os.path.join(self.obs_meta_path, self.parent_dir, "%s/%s/%s/_service"
- % (branch_name, proj, pkg))
- if os.path.exists(proj_path):
- shutil.rmtree(proj)
- if os.system("osc co %s %s &>/dev/null" % (proj, pkg)) == 0:
- os.system("cp -f %s %s" % (service_file_path, pkg_path))
- os.chdir(pkg_path)
- os.system("osc add _service")
- os.system("osc ci -m 'modify by %s'" % self.giteeUserName)
+ service_file_path = os.path.join(self.obs_meta_path, self.multi_version_dir,
+ branch_name, proj, pkg, "_service")
+ _tmpdir = os.popen("mktemp -d").read().strip('\n')
+ pkg_tmpdir = os.path.join(_tmpdir, proj, pkg)
+ cmd = "cd %s && osc co %s %s &>/dev/null && cd -" % (_tmpdir, proj, pkg)
+ if os.system(cmd) == 0:
+ cmd = "cd %s && cp -f %s %s && osc add _service && osc ci -m 'modify by %s' && cd -" \
+ % (pkg_tmpdir, service_file_path, pkg_tmpdir, self.giteeUserName)
+ if os.system(cmd) == 0:
+ log.info("modify %s %s _service success!" % (proj, pkg))
+ shutil.rmtree(_tmpdir)
+ return 0
+ else:
+ log.error("modify %s %s _service failed!" % (proj, pkg))
+ shutil.rmtree(_tmpdir)
+ return -1
else:
- log.info("modify %s %s _service failed!" % (proj, pkg))
+ log.warning("%s %s not found" % (proj, pkg))
+ shutil.rmtree(_tmpdir)
return -1
- os.chdir(self.work_dir)
- return 0
-
+
def _modify_pkg_meta(self, proj, pkg, branch_name):
"""
change the package of the meta
return 0 or -1
"""
- if os.system("osc ls %s %s &>/dev/null" % (proj, pkg)) != 0:
- log.warning("%s %s not found" % (proj, pkg))
- return -1
- os.chdir(self.work_dir)
- file_path = os.path.join(self.obs_meta_path, self.parent_dir, "%s/%s/%s/.osc/_meta"
- % (branch_name, proj, pkg))
- cmd = "osc meta pkg %s %s --file=%s | grep ^Done." % (proj, pkg, file_path)
+ file_path = os.path.join(self.obs_meta_path, self.multi_version_dir,
+ branch_name, proj, pkg, ".osc/_meta")
+ _tmpdir = os.popen("mktemp -d").read().strip('\n')
+ cmd = "cd %s && osc meta pkg %s %s --file=%s | grep ^Done. && cd -" % (_tmpdir, proj, pkg, file_path)
if os.system(cmd) != 0:
- log.error("%s/%s/.osc/_meta deal error" % (proj, pkg))
+ log.error("modify %s %s _meta failed!" % (proj, pkg))
+ shutil.rmtree(_tmpdir)
return -1
- return 0
-
+ shutil.rmtree(_tmpdir)
+
def _change_pkg_prj(self, proj, new_proj, pkg, branch_name):
"""
change the package of the project
@@ -295,7 +263,7 @@ class OBSPkgManager(object):
log.info("line:%s" % line)
new_proj = ''
new_file_path = ''
- parent_dir = ''
+ multi_version_dir = ''
log_list = list(line.split())
temp_log_type = log_list[0]
file_path = log_list[1]
@@ -303,7 +271,7 @@ class OBSPkgManager(object):
new_file_path = list(line.split())[2]
tmp_str = str(file_path).split('/')
if len(tmp_str) == 5:
- parent_dir, branch_name, proj, pkg, file_name = tmp_str
+ multi_version_dir, branch_name, proj, pkg, file_name = tmp_str
else:
branch_name, proj, pkg, file_name = str(file_path).split('/')
if len(new_file_path) != 0:
@@ -322,7 +290,7 @@ class OBSPkgManager(object):
if temp_log_type == "A":
log_type = "Add-pkg"
elif temp_log_type == "D":
- pkg_path = os.path.join(self.obs_meta_path, parent_dir, '%s/%s/%s' % (branch_name, proj, pkg))
+ pkg_path = os.path.join(self.obs_meta_path, multi_version_dir, '%s/%s/%s' % (branch_name, proj, pkg))
if os.path.exists(pkg_path):
log_type = "Del-pkg-service"
else:
@@ -336,41 +304,82 @@ class OBSPkgManager(object):
log.error("%s failed" % line)
else:
log.error("%s failed" % line)
- mesg_list = [log_type, branch_name, proj, pkg, new_proj, parent_dir]
+ mesg_list = [log_type, branch_name, proj, pkg, new_proj, multi_version_dir]
return mesg_list
-
- def _deal_some_param(self):
+
+ def _obs_pkgs_action(self):
+ """
+ operate on packages
+ """
+ file_content = []
+ data = open(self.patch_file_path, 'r')
+ for line in data:
+ file_content.append(line.strip('\n'))
+ data.close()
+ with ThreadPoolExecutor(10) as executor:
+ for content in file_content:
+ executor.submit(self._obs_pkg_action, content)
+
+ def _obs_pkg_action(self, file_content):
+ """
+ operate on a package
+ file_content: a line of the patch
+ """
+ msg = self._deal_some_param(file_content)
+ self._action_type(msg)
+
+ def _action_type(self, msg):
+ """
+ operate on the package according to the log_type
+ """
+ self.multi_version_dir = msg["multi_version_dir"]
+ if msg["log_type"] == "Add-pkg":
+ if msg["exist_flag"] == 0:
+ ret = self._add_pkg(msg["proj"], msg["pkg"], msg["branch_name"])
+ if self.sync_code and ret != -1:
+ self._sync_pkg_code(msg["proj"], msg["pkg"], msg["branch_name"])
+ elif msg["log_type"] == "Del-pkg":
+ self._del_pkg(msg["proj"], msg["pkg"])
+ elif msg["log_type"] == "Del-pkg-service":
+ self._del_obs_pkg_service(msg["proj"], msg["pkg"])
+ elif msg["log_type"] == "Mod-pkg-service":
+ self._modify_pkg_service(msg["proj"], msg["pkg"], msg["branch_name"])
+ elif msg["log_type"] == "Mod-pkg-meta":
+ self._modify_pkg_meta(msg["proj"], msg["pkg"], msg["branch_name"])
+ elif msg["log_type"] == "Change-pkg-prj":
+ self._change_pkg_prj(msg["proj"], msg["new_proj"], msg["pkg"], msg["branch_name"])
+ if self.sync_code:
+ self._sync_pkg_code(msg["new_proj"], msg["pkg"], msg["branch_name"])
+
+ def _deal_some_param(self, file_content):
"""
deal with some data and relation
+ return a dict
"""
pattern_string = ['.meta', '.prjconf', '/_service', '/_meta']
proj_list = os.listdir(os.path.join(self.obs_meta_path, "master"))
proj_list.remove("openEuler:Mainline:RISC-V")
for pattern in pattern_string:
- data = open(self.patch_file_path, 'r')
- for line in data:
- cmd1 = 'echo "%s" | grep "%s$"' % (line.strip(), pattern)
- if os.popen(cmd1).read():
- tmp = {}
- line = line.strip('\n')
- log_type, branch_name, proj, pkg, new_proj, parent_dir = self._parse_git_log(line)
- tmp["log_type"] = log_type
- tmp["branch_name"] = branch_name
- tmp["proj"] = proj
- tmp["pkg"] = pkg
- tmp["new_proj"] = new_proj
- tmp["exist_flag"] = 0
- tmp["parent_dir"] = parent_dir
- for p in proj_list:
- cmd3 = "osc ls %s 2>&1 | grep -q -Fx %s" % (p, pkg)
- if os.system(cmd3) == 0:
- if p == proj:
- log.info("package %s hava existed in obs project %s" % (pkg, p))
- tmp["exist_flag"] = 1
- self.import_list.append(tmp)
- else:
- continue
- data.close()
+ cmd = 'echo "%s" | grep "%s$"' % (file_content.strip(), pattern)
+ if os.popen(cmd).read():
+ tmp = {}
+ log_type, branch_name, proj, pkg, new_proj, multi_version_dir = self._parse_git_log(file_content)
+ tmp["log_type"] = log_type
+ tmp["branch_name"] = branch_name
+ tmp["proj"] = proj
+ tmp["pkg"] = pkg
+ tmp["new_proj"] = new_proj
+ tmp["exist_flag"] = 0
+ tmp["multi_version_dir"] = multi_version_dir
+ for p in proj_list:
+ cmd = "osc ls %s 2>&1 | grep -q -Fx %s" % (p, pkg)
+ if os.system(cmd) == 0:
+ if p == proj:
+ log.info("package %s hava existed in obs project %s" % (pkg, p))
+ tmp["exist_flag"] = 1
+ return tmp
+ else:
+ continue
def obs_pkg_admc(self):
"""
@@ -383,28 +392,7 @@ class OBSPkgManager(object):
self._check_obs_meta_pkg()
self._pre_env()
self._git_clone("obs_meta")
- self._deal_some_param()
- log.info(self.import_list)
- for msg in self.import_list:
- self.parent_dir = msg["parent_dir"]
- if msg["log_type"] == "Add-pkg":
- if msg["exist_flag"] == 0:
- ret = self._add_pkg(msg["proj"], msg["pkg"], msg["branch_name"])
- if self.sync_code and ret != -1:
- self._sync_pkg_code(msg["proj"], msg["pkg"], msg["branch_name"])
- elif msg["log_type"] == "Del-pkg":
- self._del_pkg(msg["proj"], msg["pkg"])
- elif msg["log_type"] == "Del-pkg-service":
- self._del_obs_pkg_service(msg["proj"], msg["pkg"])
- elif msg["log_type"] == "Mod-pkg-service":
- self._modify_pkg_service(msg["proj"], msg["pkg"], msg["branch_name"])
- elif msg["log_type"] == "Mod-pkg-meta":
- self._modify_pkg_meta(msg["proj"], msg["pkg"], msg["branch_name"])
- elif msg["log_type"] == "Change-pkg-prj":
- self._change_pkg_prj(msg["proj"], msg["new_proj"], msg["pkg"], msg["branch_name"])
- if self.sync_code:
- self._sync_pkg_code(msg["new_proj"], msg["pkg"], msg["branch_name"])
- return 0
+ self._obs_pkgs_action()
def _parse_yaml_data(self):
"""