From 63e50ccbae5bf75932e5f792290dea90c07361d2 Mon Sep 17 00:00:00 2001 From: liuqi <469227928@qq.com> Date: Thu, 6 Apr 2023 17:58:57 +0800 Subject: [PATCH 1/8] feature of package contribution service --- src/build/check_consistency.py | 40 ++++++++++++ src/build/gitee_comment.py | 74 ++++++---------------- src/build/osc_build_k8s.py | 108 +++++---------------------------- src/lib/comment.sh | 8 +-- src/lib/main_process.sh | 4 +- 5 files changed, 80 insertions(+), 154 deletions(-) create mode 100644 src/build/check_consistency.py diff --git a/src/build/check_consistency.py b/src/build/check_consistency.py new file mode 100644 index 0000000..bd65fdd --- /dev/null +++ b/src/build/check_consistency.py @@ -0,0 +1,40 @@ +import argparse +import json +import os +import sys +from src.proxy.kafka_proxy import KafkaProducerProxy + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', '--pkg_name', help='package name', required=True) + parser.add_argument('-sp', '--spec_url', help='spec url', required=True) + parser.add_argument('-b', '--build_url', help='build url', required=True) + parser.add_argument('-i', '--pkg_id', help='package id', required=True) + parser.add_argument('-sv', '--service', help='service info', required=True) + parser.add_argument('-bn', '--build_number', help='build number', required=True) + res = parser.parse_args() + return res + + +if __name__ == '__main__': + args = parse_args() + spec_name = args.spec_url.split('/')[-1].split('.spec')[0] + if args.pkg_name != spec_name: + comment = """ +
Check Name Build Result Build Details
check_consistency :x:FAILED #{}
""".format( + args.build_url, args.build_number) + msg = json.dumps({ + "pkg_id": args.pkg_id, + "success": False, + "detail": comment + }) + if 'test' in args.service: + topic = 'software_pkg_ci_checked_test' + else: + topic = 'software_pkg_ci_checked' + kp = KafkaProducerProxy(brokers=os.getenv('KAFKAURL')) + kp.send(topic, 'body', json.dumps(msg).encode('utf-8')) + print('Fail to check consistency of package name. Exit...') + sys.exit(1) + diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 9fc35b2..ada3d9a 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -16,6 +16,7 @@ # ********************************************************************************** """ +import base64 import os import re import stat @@ -44,16 +45,13 @@ class Comment(object): :param pr: pull request number """ - self._pr = pr self._check_item_comment_files = check_item_comment_files self._up_builds = [] self._up_up_builds = [] self._get_upstream_builds(jenkins_proxy) - self.ac_result = {} - self.compare_package_result = {} self.check_item_result = {} - def comment_build(self, gitee_proxy): + def comment_build(self): """ 构建结果 :param jenkins_proxy: @@ -61,7 +59,6 @@ class Comment(object): :return: """ comments = self._comment_build_html_format() - gitee_proxy.comment_pr(self._pr, "\n".join(comments)) return "\n".join(comments) @@ -121,9 +118,6 @@ class Comment(object): """ comments = ["", self.comment_html_table_th()] - if self._up_up_builds: - logger.debug("get up_up_builds") - comments.extend(self._comment_of_ac(self._up_up_builds[0])) if self._up_builds: comments.extend(self._comment_of_check_item(self._up_builds)) @@ -425,55 +419,27 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") - dd = DistDataset() - dd.set_attr_stime("comment.job.stime") - - # gitee pr tag - gp = GiteeProxy(args.owner, args.repo, args.gitee_token) - gp.delete_tag_of_pr(args.pr, "ci_processing") - jp = JenkinsProxy(args.jenkins_base_url, args.jenkins_user, args.jenkins_api_token) url, build_time, reason = jp.get_job_build_info(os.environ.get("JOB_NAME"), int(os.environ.get("BUILD_ID"))) - dd.set_attr_ctime("comment.job.ctime", build_time) - dd.set_attr("comment.job.link", url) - dd.set_attr("comment.trigger.reason", reason) - - dd.set_attr_stime("comment.build.stime") comment = Comment(args.pr, jp, *args.check_item_comment_files) \ if args.check_item_comment_files else Comment(args.pr, jp) logger.info("comment: build result......") - comment_content = comment.comment_build(gp) - dd.set_attr_etime("comment.build.etime") - dd.set_attr("comment.build.content.html", comment_content) - - if comment.check_build_result() == SUCCESS: - gp.delete_tag_of_pr(args.pr, "ci_failed") - gp.create_tags_of_pr(args.pr, "ci_successful") - dd.set_attr("comment.build.tags", ["ci_successful"]) - dd.set_attr("comment.build.result", "successful") - if args.check_result_file: - comment.comment_compare_package_details(gp, args.check_result_file) - else: - gp.delete_tag_of_pr(args.pr, "ci_successful") - gp.create_tags_of_pr(args.pr, "ci_failed") - dd.set_attr("comment.build.tags", ["ci_failed"]) - dd.set_attr("comment.build.result", "failed") - if args.owner != "openeuler": - comment.get_all_result_to_kafka(url) - - logger.info("comment: at committer......") - comment.comment_at(args.committer, gp) - - dd.set_attr_etime("comment.job.etime") - - # suppress python warning - warnings.filterwarnings("ignore") - logging.getLogger("elasticsearch").setLevel(logging.WARNING) - logging.getLogger("kafka").setLevel(logging.WARNING) - - # upload to es - kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) - query = {"term": {"id": args.comment_id}} - script = {"lang": "painless", "source": "ctx._source.comment = params.comment", "params": dd.to_dict()} - kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) + comment_content = comment.comment_build() + success = True + for i in comment_content.split('\n')[2:-1]: + if 'check_build' in i and ':x:FAILED' in i: + success = False + break + body_content = { + "pkg_id": os.getenv('pkg_id'), + "success": success, + "detail": comment_content.replace('\n', '') + } + body = base64.b64encode(json.dumps(body_content).encode('utf-8')).decode() + msg = { + "Header": None, + "Body": body + } + kp = KafkaProducerProxy(brokers=os.getenv('KAFKAURL')) + kp.send('software_pkg_ci_checked_test', '', msg) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 99d7a46..651e476 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -69,15 +69,6 @@ class SinglePackageBuild(object): logger.info("checkout ... ok") - # update package meta file "_service" - self._handle_package_meta(project, work_dir, code_dir) - logger.debug("prepare \"_service\" ... ok") - - # process_service.pl - if not self._prepare_build_environ(project, work_dir): - logger.error("prepare environ ... failed") - return 2 - logger.info("prepare environ ... ok") # osc build @@ -191,40 +182,22 @@ class SinglePackageBuild(object): if self._branch.lower() in Constant.STOPPED_MAINTENANCE_BRANCH: logger.error("branch \"%s\" is no longer maintained!", self._branch) return 1 - if self._branch not in Constant.GITEE_BRANCH_PROJECT_MAPPING: - logger.error("branch \"%s\" not support yet", self._branch) - return 1 has_any_repo_build = False - for project in Constant.GITEE_BRANCH_PROJECT_MAPPING.get(self._branch): - logger.debug("start build project %s", project) - - obs_repos = self.get_need_build_obs_repos(project) - if not obs_repos: - logger.info("all repos ignored of project %s", project) - continue - logger.debug("build obs repos: %s", obs_repos) - - logger.info("project: %s", project) - with open("obs_project", "w") as f: - f.write(project) - - has_any_repo_build = True - ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) - if ret > 0: - logger.debug("build run return %s", ret) - logger.error("build %s %s %s ... %s", project, self._package, self._arch, "failed") - return 1 # finish if any error - else: - logger.info("build %s %s %s ... %s", project, self._package, self._arch, "ok") - break - - # if no repo build, regard as fail - if not has_any_repo_build: - logger.error("package not in any obs projects, please add package into obs") - return 1 - - return 0 + project = 'home:nicliuqi' + obs_repos = [{ + 'repo': 'standard_' + self._arch.replace('-', '_'), + 'mpac': os.getenv('package'), + 'state': 'succeeded*' + }] + ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) + if ret > 0: + logger.debug("build run return %s", ret) + logger.error("build %s %s %s ... %s", project, self._package, self._arch, "failed") + return 1 # finish if any error + else: + logger.info("build %s %s %s ... %s", project, self._package, self._arch, "ok") + return 0 def init_args(): @@ -259,64 +232,15 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") - logger.info("using credential %s", args.account.split(":")[0]) - logger.info("cloning repository https://gitee.com/%s/%s.git ", args.owner, args.repo) - logger.info("clone depth 1") - logger.info("checking out pull request %s", args.pr) + logger.info('osc buils k8s start') from src.utils.dist_dataset import DistDataset from src.proxy.git_proxy import GitProxy from src.proxy.obs_proxy import OBSProxy - from src.proxy.es_proxy import ESProxy from src.proxy.kafka_proxy import KafkaProducerProxy from src.utils.shell_cmd import shell_cmd_live - dd = DistDataset() - dd.set_attr_stime("spb.job.stime") - dd.set_attr("spb.job.link", os.environ["BUILD_URL"]) - dd.set_attr("spb.trigger.reason", os.environ["BUILD_CAUSE"]) - - # suppress python warning - warnings.filterwarnings("ignore") - logging.getLogger("elasticsearch").setLevel(logging.WARNING) - logging.getLogger("kafka").setLevel(logging.WARNING) - - kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) - - # download repo - dd.set_attr_stime("spb.scm.stime") - gp = GitProxy.init_repository(args.repo, work_dir=args.workspace) - repo_url = "https://{}@gitee.com/{}/{}.git".format(args.account, args.owner, args.repo) - if not gp.fetch_pull_request(repo_url, args.pr, depth=1): - logger.info("fetch finished -") - - dd.set_attr("spb.scm.result", "failed") - dd.set_attr_etime("spb.scm.etime") - dd.set_attr_etime("spb.job.etime") - #dd.set_attr("spb.job.result", "failed") - - # upload to es - query = {"term": {"id": args.comment_id}} - script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), - "params": dd.to_dict()} - kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) - sys.exit(-1) - else: - gp.checkout_to_commit_force("pull/{}/MERGE".format(args.pr)) - logger.info("fetch finished +") - dd.set_attr("spb.scm.result", "successful") - dd.set_attr_etime("spb.scm.etime") - - dd.set_attr_stime("spb.build.stime") spb = SinglePackageBuild(args.package, args.arch, args.branch) rs = spb.build(args.spec, args.workspace, args.code) - dd.set_attr("spb.build.result", "failed" if rs else "successful") - dd.set_attr_etime("spb.build.etime") - - dd.set_attr_etime("spb.job.etime") - - # upload to es - query = {"term": {"id": args.comment_id}} - script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) + logger.info('osc build k8s ends') sys.exit(rs) diff --git a/src/lib/comment.sh b/src/lib/comment.sh index 62005d9..bf7b0a3 100644 --- a/src/lib/comment.sh +++ b/src/lib/comment.sh @@ -71,14 +71,10 @@ function scp_comment_file() { function exec_comment() { log_info "***** Start to exec comment *****" export PYTHONPATH=${shell_path} - python3 ${shell_path}/src/build/gitee_comment.py -o $repo_owner -r $repo -p $prid -c $committer -t ${giteetoken}\ - -b $jenkins_api_host -u $jenkins_user -j $jenkins_api_token -a ${check_item_comment_aarch64} ${check_item_comment_x86}\ - -f ${compare_package_result_x86},${compare_package_result_aarch64} -m ${commentid} + python3 ${shell_path}/src/build/gitee_comment.py -b $jenkins_api_host -u $jenkins_user -j $jenkins_api_token \ + -a ${check_item_comment_aarch64} ${check_item_comment_x86} log_info "***** End to exec comment *****" log_info "***** Start to exec comment to kafka*****" - python3 ${shell_path}/src/build/comment_to_dashboard.py -r $repo -c $committer -m ${commentid} -g $jobtriggertime\ - -k "${prtitle}" -t $prcreatetime -b $tbranch -u $prurl -i $triggerbuildid -p $prid -o $repo_owner \ - --gitee_token ${giteetoken} log_info "***** End to exec comment to kafka*****" } diff --git a/src/lib/main_process.sh b/src/lib/main_process.sh index c4caac7..62eefb7 100644 --- a/src/lib/main_process.sh +++ b/src/lib/main_process.sh @@ -184,6 +184,8 @@ function build_packages() { log_info "***** Start to build package *****" comment_file="${repo}_${prid}_${arch}_comment" export PYTHONPATH=${shell_path} + python3 ${SCRIPT_PATCH}/osc_build_k8s.py -o ${repo_owner} -p ${package} -a $arch -c $WORKSPACE -b $tbranch -r ${repo} -m ${commentid} --pr ${prid} -t ${GiteeUserPassword} + python3 ${SCRIPT_PATCH}/extra_work.py checkinstall -a ${arch} -r $tbranch --obs_rpm_host ${obs_rpm_host} --install-root=${WORKSPACE}/install_root/${commentid} -e $WORKSPACE/${comment_file} || echo "continue although run check install failed" for item in $(echo ${package} | sed 's/,/ /g'); do log_info "start build package $item" log_debug "params are [$repo, $branch, $prid, $committer, $arch, $package, $buddy, $WORKSPACE]" @@ -330,7 +332,6 @@ EOF # 执行入口 function main() { - config_osc config_ipv6 config_maven config_gradle @@ -349,6 +350,5 @@ function main() { if [[ $exclusive_arch ]]; then log_info "exclusive_arch not empty" build_packages - compare_package fi } -- Gitee From ec6a2575a17d303cab6ad0c1ab0f4b3afd572780 Mon Sep 17 00:00:00 2001 From: liuqi <469227928@qq.com> Date: Thu, 6 Apr 2023 20:08:09 +0800 Subject: [PATCH 2/8] modify message format of Kafka --- src/build/check_consistency.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/build/check_consistency.py b/src/build/check_consistency.py index bd65fdd..48e874a 100644 --- a/src/build/check_consistency.py +++ b/src/build/check_consistency.py @@ -1,4 +1,5 @@ import argparse +import base64 import json import os import sys @@ -24,17 +25,23 @@ if __name__ == '__main__': comment = """
Check Name Build Result Build Details
check_consistency :x:FAILED #{}
""".format( args.build_url, args.build_number) - msg = json.dumps({ - "pkg_id": args.pkg_id, - "success": False, - "detail": comment - }) + body_content = { + 'pkg_id': os.getenv('pkg_id'), + 'success': False, + 'detail': comment + } + body = base64.b64encode(json.dumps(body_content).encode('utf-8')).decode() + msg = { + 'Header': None, + 'Body': body + } + if 'test' in args.service: topic = 'software_pkg_ci_checked_test' else: topic = 'software_pkg_ci_checked' kp = KafkaProducerProxy(brokers=os.getenv('KAFKAURL')) - kp.send(topic, 'body', json.dumps(msg).encode('utf-8')) + kp.send(topic, '', msg) + print('Error report had been sent.') print('Fail to check consistency of package name. Exit...') sys.exit(1) - -- Gitee From ceefe3cfdfb5445faa8e354cf69be8a834f1ac85 Mon Sep 17 00:00:00 2001 From: liuqi <469227928@qq.com> Date: Fri, 7 Apr 2023 11:30:57 +0800 Subject: [PATCH 3/8] fix issue of markdown display --- src/build/check_consistency.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/build/check_consistency.py b/src/build/check_consistency.py index 48e874a..a687d88 100644 --- a/src/build/check_consistency.py +++ b/src/build/check_consistency.py @@ -22,9 +22,9 @@ if __name__ == '__main__': args = parse_args() spec_name = args.spec_url.split('/')[-1].split('.spec')[0] if args.pkg_name != spec_name: - comment = """ -
Check Name Build Result Build Details
check_consistency :x:FAILED #{}
""".format( - args.build_url, args.build_number) + comment = '
Check Name Build Result Build ' \ + 'Details
check_consistency :x:FAILED #{}
'.format(args.build_url, args.build_number) body_content = { 'pkg_id': os.getenv('pkg_id'), 'success': False, -- Gitee From c21bc5034e5b0d51b64ee1fb0dacd0bc0d4242d7 Mon Sep 17 00:00:00 2001 From: liuqi <469227928@qq.com> Date: Fri, 7 Apr 2023 16:02:56 +0800 Subject: [PATCH 4/8] add download for gitee source file and add kafka topic --- src/build/gitee_comment.py | 5 ++++- src/utils/download.py | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 src/utils/download.py diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index ada3d9a..6ad02bc 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -442,4 +442,7 @@ if "__main__" == __name__: "Body": body } kp = KafkaProducerProxy(brokers=os.getenv('KAFKAURL')) - kp.send('software_pkg_ci_checked_test', '', msg) + if 'test' in os.getenv('service'): + kp.send('software_pkg_ci_checked_test', '', msg) + else: + kp.send('software_pkg_ci_checked', '', msg) diff --git a/src/utils/download.py b/src/utils/download.py new file mode 100644 index 0000000..e00662a --- /dev/null +++ b/src/utils/download.py @@ -0,0 +1,38 @@ +""" +Avoid incomplete downloads due to redirection, use Gitee V5 API to get raw file. +""" +import base64 +import requests +import subprocess +import sys +from urllib.parse import urlparse + + +if len(sys.argv) < 2: + print('ERROR! Lost of required source url.') + sys.exit(1) +url = sys.argv[1] +urlparse_res = urlparse(url) +if urlparse_res.netloc != 'gitee.com': + cmd = 'wget {}'.format(url) + res = subprocess.call(cmd, shell=True) + sys.exit(res) +urlparse_path = urlparse_res.path +if len(urlparse_path.split('/')) < 6: + print('ERROR! You must apply a file address instead of a repo address.') + sys.exit(1) +_, owner, repo, fmt, branch, filepath = urlparse_path.split('/', 5) +if fmt != 'raw': + print('ERROR! Source file must be raw format.') + sys.exit(1) +request_url = 'https://gitee.com/api/v5/repos/{}/{}/contents/{}?ref={}'.format(owner, repo, filepath, branch) +r = requests.get(request_url) +if r.status_code != 200: + print('ERROR! Unexpected Error: {}'.format(r.content)) + sys.exit(1) +content_b64 = r.json()['content'] +content = base64.b64decode(content_b64) +filename = filepath.split('/')[-1] +with open(filename, 'wb') as f: + f.write(content) +print('Downlaod {} successfully.'.format(filename)) -- Gitee From bf515e6aa1008022646ecf0203a0137319c3d003 Mon Sep 17 00:00:00 2001 From: liuqi <469227928@qq.com> Date: Fri, 7 Apr 2023 17:09:33 +0800 Subject: [PATCH 5/8] add string split to support multi addresses --- src/build/gitee_comment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 6ad02bc..88a6c39 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -441,7 +441,7 @@ if "__main__" == __name__: "Header": None, "Body": body } - kp = KafkaProducerProxy(brokers=os.getenv('KAFKAURL')) + kp = KafkaProducerProxy(brokers=os.getenv('KAFKAURL').split(',')) if 'test' in os.getenv('service'): kp.send('software_pkg_ci_checked_test', '', msg) else: -- Gitee From 9ac0800e8ef2d2dce96d4b73e0732e48a6fc0728 Mon Sep 17 00:00:00 2001 From: liuqi <469227928@qq.com> Date: Mon, 10 Apr 2023 19:16:12 +0800 Subject: [PATCH 6/8] add access token to params --- src/utils/download.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/utils/download.py b/src/utils/download.py index e00662a..abae075 100644 --- a/src/utils/download.py +++ b/src/utils/download.py @@ -8,10 +8,11 @@ import sys from urllib.parse import urlparse -if len(sys.argv) < 2: - print('ERROR! Lost of required source url.') +if len(sys.argv) < 3: + print('ERROR! Lost of required source url and access token.') sys.exit(1) url = sys.argv[1] +access_token = sys.argv[2] urlparse_res = urlparse(url) if urlparse_res.netloc != 'gitee.com': cmd = 'wget {}'.format(url) @@ -25,8 +26,12 @@ _, owner, repo, fmt, branch, filepath = urlparse_path.split('/', 5) if fmt != 'raw': print('ERROR! Source file must be raw format.') sys.exit(1) -request_url = 'https://gitee.com/api/v5/repos/{}/{}/contents/{}?ref={}'.format(owner, repo, filepath, branch) -r = requests.get(request_url) +request_url = 'https://gitee.com/api/v5/repos/{}/{}/contents/{}'.format(owner, repo, filepath) +params = { + 'branch': branch, + 'access_token': access_token +} +r = requests.get(request_url, params=params) if r.status_code != 200: print('ERROR! Unexpected Error: {}'.format(r.content)) sys.exit(1) -- Gitee From 0136d2f4184a559d19fd270d71cba01c1ee79c84 Mon Sep 17 00:00:00 2001 From: liuqi <469227928@qq.com> Date: Tue, 11 Apr 2023 11:20:56 +0800 Subject: [PATCH 7/8] optimize download logic --- src/utils/download.py | 100 ++++++++++++++++++++++++++++-------------- 1 file changed, 67 insertions(+), 33 deletions(-) diff --git a/src/utils/download.py b/src/utils/download.py index abae075..16198f8 100644 --- a/src/utils/download.py +++ b/src/utils/download.py @@ -2,42 +2,76 @@ Avoid incomplete downloads due to redirection, use Gitee V5 API to get raw file. """ import base64 +import os import requests import subprocess import sys from urllib.parse import urlparse -if len(sys.argv) < 3: - print('ERROR! Lost of required source url and access token.') - sys.exit(1) -url = sys.argv[1] -access_token = sys.argv[2] -urlparse_res = urlparse(url) -if urlparse_res.netloc != 'gitee.com': - cmd = 'wget {}'.format(url) - res = subprocess.call(cmd, shell=True) - sys.exit(res) -urlparse_path = urlparse_res.path -if len(urlparse_path.split('/')) < 6: - print('ERROR! You must apply a file address instead of a repo address.') - sys.exit(1) -_, owner, repo, fmt, branch, filepath = urlparse_path.split('/', 5) -if fmt != 'raw': - print('ERROR! Source file must be raw format.') - sys.exit(1) -request_url = 'https://gitee.com/api/v5/repos/{}/{}/contents/{}'.format(owner, repo, filepath) -params = { - 'branch': branch, - 'access_token': access_token -} -r = requests.get(request_url, params=params) -if r.status_code != 200: - print('ERROR! Unexpected Error: {}'.format(r.content)) - sys.exit(1) -content_b64 = r.json()['content'] -content = base64.b64decode(content_b64) -filename = filepath.split('/')[-1] -with open(filename, 'wb') as f: - f.write(content) -print('Downlaod {} successfully.'.format(filename)) +def repo_branches(owner, repo, access_token): + request_url = 'https://gitee.com/api/v5/repos/{}/{}/branches'.format(owner, repo) + params = { + 'access_token': access_token + } + r = requests.get(request_url, params=params) + if r.status_code != 200: + print('ERROR! Unexpected Error: {}'.format(r.content)) + sys.exit(1) + return [x['name'] for x in r.json()] + + +def repo_file_content(owner, repo, branch, access_token, filepath): + request_url = 'https://gitee.com/api/v5/repos/{}/{}/contents/{}'.format(owner, repo, filepath) + params = { + 'ref': branch, + 'access_token': access_token + } + r = requests.get(request_url, params=params) + if r.status_code == 404: + return '' + elif r.status_code != 200: + print('ERROR! Unexpected Error: {}'.format(r.content)) + sys.exit(1) + return r.json()['content'] + + +def main(): + if len(sys.argv) < 3: + print('ERROR! Lost of required source url and access token.') + sys.exit(1) + url = sys.argv[1] + access_token = sys.argv[2] + urlparse_res = urlparse(url) + if urlparse_res.netloc != 'gitee.com': + cmd = 'wget {}'.format(url) + res = subprocess.call(cmd, shell=True) + sys.exit(res) + urlparse_path = urlparse_res.path + if len(urlparse_path.split('/')) < 6: + print('ERROR! You must apply a file address instead of a repo address.') + sys.exit(1) + _, owner, repo, fmt, branch_filepath = urlparse_path.split('/', 4) + filename = branch_filepath.split('/')[-1] + if fmt != 'raw': + print('ERROR! Source file must be raw format.') + sys.exit(1) + branches = repo_branches(owner, repo, access_token) + for branch in branches: + if branch_filepath.startswith(branch) and len(branch_filepath.split(branch)) == 2: + filepath = branch_filepath.split(branch + '/')[1] + content_b64 = repo_file_content(owner, repo, branch, access_token, filepath) + if not content_b64: + continue + content = base64.b64decode(content_b64) + with open(filename, 'wb') as f: + f.write(content) + print('Download {} successfully.'.format(filename)) + break + if not os.path.exists(filename): + print('Download {} failed.'.format(filename)) + sys.exit(1) + + +if __name__ == '__main__': + main() -- Gitee From bb4c512b7be1aba2117432bab147bbd683cbc9a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=99=E5=98=89=E7=90=A6?= Date: Wed, 12 Apr 2023 06:56:15 +0000 Subject: [PATCH 8/8] update src/ac/acl/commit_msg/check_commit_msg.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 孙嘉琦 --- src/ac/acl/commit_msg/check_commit_msg.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/ac/acl/commit_msg/check_commit_msg.py b/src/ac/acl/commit_msg/check_commit_msg.py index 7257d7a..70de40d 100644 --- a/src/ac/acl/commit_msg/check_commit_msg.py +++ b/src/ac/acl/commit_msg/check_commit_msg.py @@ -52,8 +52,9 @@ class CheckCommitMsg(BaseCheck): commit = commit.replace('\r', '') commit = commit.replace('\n', '') # get commit and add to dist for follow using - command = "gitlint --commit " + commit + " -C " + gitlint_dir + "/.gitlint" - res = subprocess.Popen(command, cwd=work_dir, shell=True, stderr=subprocess.PIPE) + command = "--commit " + commit + " -C " + gitlint_dir + "/.gitlint" + command_list = ["gitlint", command] + res = subprocess.Popen(command_list, cwd=work_dir, shell=False, stderr=subprocess.PIPE) out = res.stderr.readlines() if len(out) > 0 : out_str = "" @@ -71,12 +72,14 @@ class CheckCommitMsg(BaseCheck): # prepare git branch environment repo_dir = self._work_dir + "/" + self._repo logger.info("repoDir: %s", repo_dir) - branch_log_cmd = "git fetch origin +refs/heads/{}:refs/remotes/origin/{}".format(self._tbranch, self._tbranch) - branch_log_pro = subprocess.Popen(branch_log_cmd, cwd=repo_dir, shell=True, stdout=subprocess.PIPE) + branch_log_cmd = "origin +refs/heads/{}:refs/remotes/origin/{}".format(self._tbranch, self._tbranch) + branch_log_cmd_list = ["git fetch", branch_log_cmd] + branch_log_pro = subprocess.Popen(branch_log_cmd_list, cwd=repo_dir, shell=False, stdout=subprocess.PIPE) logger.info("git featch res: ") logger.info(branch_log_pro.stdout.read()) - branch_log_cmd = "git log HEAD...origin/" + self._tbranch + " --no-merges --pretty=%H" - branch_log_pro = subprocess.Popen(branch_log_cmd, cwd=repo_dir, shell=True, stdout=subprocess.PIPE) + branch_log_cmd = "HEAD...origin/" + self._tbranch + " --no-merges --pretty=%H" + branch_log_cmd_list = ["git log", branch_log_cmd] + branch_log_pro = subprocess.Popen(branch_log_cmd_list, cwd=repo_dir, shell=False, stdout=subprocess.PIPE) branch_log_res = branch_log_pro.stdout.readlines() conf_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../../conf/")) commit_check_dist = self.get_commit_msg_result(branch_log_res, repo_dir, conf_dir) -- Gitee