diff --git a/common/common.py b/common/common.py index c710431f0634186d76b0c6374665c557fe82af12..1a9d4433005b8b5cbb18ccb5de54da6132a4ad2b 100644 --- a/common/common.py +++ b/common/common.py @@ -22,6 +22,8 @@ import re import pexpect import requests import jenkins +import subprocess +from common.log_obs import log from requests.auth import HTTPBasicAuth try: from urllib import urlencode @@ -35,6 +37,22 @@ def str_to_bool(s): """ return s.lower() in ("yes", "true", "t", "1") +def run(cmd, timeout=600, print_out=False): + """ + run shell cmd + :param cmd: command + :param timeout: timeout + :param print_out: print out or not + :return: return code, stdout, stderr + """ + ret = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, encoding="utf-8", timeout=timeout) + log.info("cmd: {}".format(cmd)) + if ret.stdout and print_out: + log.info("ret.stdout: {}".format(ret.stdout)) + if ret.stderr: + log.warning("ret.stderr: {}".format(ret.stderr)) + return ret.returncode, ret.stdout, ret.stderr def git_repo_src(repo_url, gitee_user_name, gitee_user_pwd, dest_dir=None): """ diff --git a/core/getdate.py b/core/getdate.py index 804fab294537ddada74d00e65d7808b16c857eea..3472b4efe9513752d08ddbfd8cc55a48fb04bdc0 100644 --- a/core/getdate.py +++ b/core/getdate.py @@ -18,13 +18,15 @@ get the latest date to obs_pkg_rpms """ import os import sys +import shutil import calendar -import subprocess current_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) sys.path.append(os.path.join(current_path, "..")) -from common.common import Pexpect +from common.common import Pexpect, run from common.log_obs import log from concurrent.futures import ThreadPoolExecutor +from common.parser_config import ParserConfigIni + class GETDate(object): """ @@ -49,6 +51,9 @@ class GETDate(object): self.giteeuser = kwargs["gitee_user"] self.giteeuserpwd = kwargs["gitee_pwd"] self.sourcedir = "/srv/cache/obs/tar_scm/repo/next/" + par = ParserConfigIni() + self.obs_pkg_rpms_url = par.get_repos_dict()["obs_pkg_rpms"] + self.gitee_repo_name = self.obs_pkg_rpms_url.split('/')[-1] self.pkg_time = {} self.realdir = None @@ -74,8 +79,9 @@ class GETDate(object): cmd_result = self.cmd.ssh_cmd(cmd) log.info("ssh_cmd_for_ls: %s" % cmd_result) for rpm in cmd_result: - if rpm.decode("utf-8").replace('\r\n', '') != ": ": - rpmlist.append(rpm.decode("utf-8").replace('\r\n', '')) + tmp = rpm.decode("utf-8").replace('\r\n', '') + if tmp != ": ": + rpmlist.append(tmp) log.info("rpmlist: %s", ",".join(rpmlist)) return rpmlist @@ -94,20 +100,19 @@ class GETDate(object): get a fixed format date rpm: which package you want to get """ - cmd = "show -s --format=%ad --date=local" pkg_dir = os.path.join(self.realdir, rpm) - show_result = self.cmd.ssh_cmd("git -C %s %s" % (pkg_dir, cmd)) + res = "show -s --format=%ad --date=local" + cmd = f"git -C {pkg_dir} {res}" + show_result = self.cmd.ssh_cmd(cmd) log.info("%s : %s" % (rpm, show_result[1].decode("utf-8").replace('\r\n', ''))) - str_day = show_result[1].decode("utf-8").replace('\r\n', '').split()[2] - year = show_result[1].decode("utf-8").replace('\r\n', '').split()[4] - timestr = show_result[1].decode("utf-8").replace('\r\n', '').split()[3] - eng_month = show_result[1].decode("utf-8").replace('\r\n', '').split()[1] - int_month = list(calendar.month_abbr).index(eng_month) - day = self._get_two_digis(int(str_day)) + result_list = show_result[1].decode("utf-8").replace('\r\n', '').split() + int_month = list(calendar.month_abbr).index(result_list[1]) month = self._get_two_digis(int_month) - log.info("year:%s month:%s day:%s timestr:%s" % (year, month, day, timestr)) - datestr = year + month + day + ' ' + timestr.replace(':', '-') - log.info(datestr) + day = self._get_two_digis(int(result_list[2])) + timestr = result_list[3].replace(':', '-') + year = result_list[4] + datestr = year + month + day + ' ' + timestr + log.info("%s : %s" % (rpm, datestr)) self.pkg_time[rpm] = datestr def _get_all_time_thread(self): @@ -119,15 +124,14 @@ class GETDate(object): with ThreadPoolExecutor(10) as executor: for pkg in rpmlist: executor.submit(self._get_latest_time, pkg) - print(self.pkg_time) + log.info(self.pkg_time) def _echo_to_git_thread(self): """ use the thread pool to echo time str """ - all_time = self.pkg_time with ThreadPoolExecutor(10) as executor: - for rpm, time in all_time.items(): + for rpm, time in self.pkg_time.items(): time_args = [rpm, time] executor.submit(self._echo_to_obs_pkg_rpms, time_args) @@ -135,22 +139,24 @@ class GETDate(object): """ clone the obs_pkg_rpms repository """ - clone_path = os.getcwd() - log.info("Git clone the obs_pkg_rpms %s" % clone_path) - remove_git = os.popen("rm -rf ./obs_pkg_rpms").read() - log.info("rm result: %s" % remove_git) - repo_url = "https://%s:%s@gitee.com/unsunghero/obs_pkg_rpms" \ - % (self.giteeuser, self.giteeuserpwd) + log.info("Git clone the %s" % self.gitee_repo_name) + repo_path = os.path.join(os.getcwd(), self.gitee_repo_name) + if os.path.exists(repo_path): + shutil.rmtree(repo_path) + tmp = self.obs_pkg_rpms_url.split("//") + repo_url = "%s//%s:%s@%s" % (tmp[0], self.giteeuser, self.giteeuserpwd, tmp[1]) clone_cmd = "git clone --depth=1 %s" % repo_url + clone_flag = False for i in range(5): - clone_result = subprocess.getstatusoutput(clone_cmd) + clone_result, _, _ = run(clone_cmd) log.info(clone_result) - if clone_result[0] == 0: - clone_flag = "yes" + if clone_result == 0: + clone_flag = True break else: - rm_repo = os.popen("rm -rf obs_pkg_rpms").read() - if clone_flag != "yes": + if os.path.exits(repo_path): + shutil.rmtree(repo_path) + if not clone_flag: raise SystemExit('Please check you internet!!!!!') def _echo_to_obs_pkg_rpms(self, all_time): @@ -158,10 +164,10 @@ class GETDate(object): echo the time str to a package all_time: the list for rpm name and timestr """ - echo_cmd = "echo %s > ./obs_pkg_rpms/%s/%s" % (all_time[1], \ - self.branch, all_time[0]) + echo_cmd = "echo %s > ./%s/%s/%s" % (all_time[1], \ + self.gitee_repo_name, self.branch, all_time[0]) log.info(echo_cmd) - cmd_result = os.system(echo_cmd) + cmd_result, _, _ = run(echo_cmd) if cmd_result != 0: raise SystemExit("This %s echo error to %s" % (all_time[0], self.branch)) @@ -169,21 +175,25 @@ class GETDate(object): """ push the latest obs_pkg_rpms to origin """ - os.chdir("obs_pkg_rpms") + os.chdir(self.gitee_repo_name) status_cmd = "git status -s" commit_cmd = "git add -A && git commit -m 'update for package'" - if os.popen(status_cmd).read(): - if os.system(commit_cmd) == 0: + _, result, _ = run(status_cmd) + if result: + result, _, _ = run(commit_cmd) + if result == 0: for i in range(5): push_cmd = "git push -f" - push_result = subprocess.getstatusoutput(push_cmd) + push_result, _, _ = run(push_cmd) log.info(push_result) - if push_result[0] == 0: - log.info("SUCCESS:Push success for latest date to obs_pkg_rpms") + if push_result == 0: + log.info("SUCCESS:Push success for latest date to %s" % self.gitee_repo_name) return else: - log.debug("Try Push to obs_pkg_rpms: %s" % i) - raise SystemExit("Failed: Push to obs_pkg_rpms failed") + log.debug("Try Push to %s: %s" % (self.gitee_repo_name, i)) + raise SystemExit("Failed: Push failed") + else: + raise SystemExit("Failed: git add and commit failed") else: log.info("NO CHAGE,nothing to commit") @@ -195,15 +205,3 @@ class GETDate(object): self._get_all_time_thread() self._echo_to_git_thread() self._push_to_pkg_rpms() - -if __name__ == "__main__": - - kw = {'branch': "master", - 'source_server_user': "root", - 'source_server_ip': "127.0.0.1", - 'source_server_pwd': "112233", - 'source_server_port': "2222", - 'gitee_user': "***", - 'gitee_pwd': "***"} - get = GETDate(**kw) - get.update_to_obs_pkg_rpms() diff --git a/core/save.py b/core/save.py index 37b432fd98be2375922bce45b6bc31593bb034e1..ffc829e473a07d12dcecc5bfecad3c99ced96317 100644 --- a/core/save.py +++ b/core/save.py @@ -112,7 +112,7 @@ class SaveInfo(object): timestr = os.popen(cmd).read().replace("\n", "") else: timestr = 0 - cmd = "osc list -b %s %s 2>/dev/null | grep rpm" % (prj, pkg) + cmd = "osc list -b %s %s 2>/dev/null | grep rpm | grep -v 'sw_64'" % (prj, pkg) log.debug(cmd) rpms = ' '.join(list(set(os.popen(cmd).read().replace(" ", "").split("\n")) - set(['']))) f_csv.writerow([timestr, pkg, rpms])