diff --git a/src/ac/acl/commit_msg/check_commit_msg.py b/src/ac/acl/commit_msg/check_commit_msg.py index 7257d7a0db1424ceae0264e2c357fe0369f1dcd4..70de40de038bafc22d007e93b9594b4e30ecacf1 100644 --- a/src/ac/acl/commit_msg/check_commit_msg.py +++ b/src/ac/acl/commit_msg/check_commit_msg.py @@ -52,8 +52,9 @@ class CheckCommitMsg(BaseCheck): commit = commit.replace('\r', '') commit = commit.replace('\n', '') # get commit and add to dist for follow using - command = "gitlint --commit " + commit + " -C " + gitlint_dir + "/.gitlint" - res = subprocess.Popen(command, cwd=work_dir, shell=True, stderr=subprocess.PIPE) + command = "--commit " + commit + " -C " + gitlint_dir + "/.gitlint" + command_list = ["gitlint", command] + res = subprocess.Popen(command_list, cwd=work_dir, shell=False, stderr=subprocess.PIPE) out = res.stderr.readlines() if len(out) > 0 : out_str = "" @@ -71,12 +72,14 @@ class CheckCommitMsg(BaseCheck): # prepare git branch environment repo_dir = self._work_dir + "/" + self._repo logger.info("repoDir: %s", repo_dir) - branch_log_cmd = "git fetch origin +refs/heads/{}:refs/remotes/origin/{}".format(self._tbranch, self._tbranch) - branch_log_pro = subprocess.Popen(branch_log_cmd, cwd=repo_dir, shell=True, stdout=subprocess.PIPE) + branch_log_cmd = "origin +refs/heads/{}:refs/remotes/origin/{}".format(self._tbranch, self._tbranch) + branch_log_cmd_list = ["git fetch", branch_log_cmd] + branch_log_pro = subprocess.Popen(branch_log_cmd_list, cwd=repo_dir, shell=False, stdout=subprocess.PIPE) logger.info("git featch res: ") logger.info(branch_log_pro.stdout.read()) - branch_log_cmd = "git log HEAD...origin/" + self._tbranch + " --no-merges --pretty=%H" - branch_log_pro = subprocess.Popen(branch_log_cmd, cwd=repo_dir, shell=True, stdout=subprocess.PIPE) + branch_log_cmd = "HEAD...origin/" + self._tbranch + " --no-merges --pretty=%H" + branch_log_cmd_list = ["git log", branch_log_cmd] + branch_log_pro = subprocess.Popen(branch_log_cmd_list, cwd=repo_dir, shell=False, stdout=subprocess.PIPE) branch_log_res = branch_log_pro.stdout.readlines() conf_dir = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../../conf/")) commit_check_dist = self.get_commit_msg_result(branch_log_res, repo_dir, conf_dir) diff --git a/src/build/check_consistency.py b/src/build/check_consistency.py new file mode 100644 index 0000000000000000000000000000000000000000..a687d887979816b1606fe910ac81718595df0dd0 --- /dev/null +++ b/src/build/check_consistency.py @@ -0,0 +1,47 @@ +import argparse +import base64 +import json +import os +import sys +from src.proxy.kafka_proxy import KafkaProducerProxy + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument('-p', '--pkg_name', help='package name', required=True) + parser.add_argument('-sp', '--spec_url', help='spec url', required=True) + parser.add_argument('-b', '--build_url', help='build url', required=True) + parser.add_argument('-i', '--pkg_id', help='package id', required=True) + parser.add_argument('-sv', '--service', help='service info', required=True) + parser.add_argument('-bn', '--build_number', help='build number', required=True) + res = parser.parse_args() + return res + + +if __name__ == '__main__': + args = parse_args() + spec_name = args.spec_url.split('/')[-1].split('.spec')[0] + if args.pkg_name != spec_name: + comment = '
Check Name Build Result Build ' \ + 'Details
check_consistency :x:FAILED #{}
'.format(args.build_url, args.build_number) + body_content = { + 'pkg_id': os.getenv('pkg_id'), + 'success': False, + 'detail': comment + } + body = base64.b64encode(json.dumps(body_content).encode('utf-8')).decode() + msg = { + 'Header': None, + 'Body': body + } + + if 'test' in args.service: + topic = 'software_pkg_ci_checked_test' + else: + topic = 'software_pkg_ci_checked' + kp = KafkaProducerProxy(brokers=os.getenv('KAFKAURL')) + kp.send(topic, '', msg) + print('Error report had been sent.') + print('Fail to check consistency of package name. Exit...') + sys.exit(1) diff --git a/src/build/gitee_comment.py b/src/build/gitee_comment.py index 9fc35b2dbbb75877d961129b3426fa5eaa4cbfff..88a6c396cd7c495064a2a629c80798b30b943af3 100755 --- a/src/build/gitee_comment.py +++ b/src/build/gitee_comment.py @@ -16,6 +16,7 @@ # ********************************************************************************** """ +import base64 import os import re import stat @@ -44,16 +45,13 @@ class Comment(object): :param pr: pull request number """ - self._pr = pr self._check_item_comment_files = check_item_comment_files self._up_builds = [] self._up_up_builds = [] self._get_upstream_builds(jenkins_proxy) - self.ac_result = {} - self.compare_package_result = {} self.check_item_result = {} - def comment_build(self, gitee_proxy): + def comment_build(self): """ 构建结果 :param jenkins_proxy: @@ -61,7 +59,6 @@ class Comment(object): :return: """ comments = self._comment_build_html_format() - gitee_proxy.comment_pr(self._pr, "\n".join(comments)) return "\n".join(comments) @@ -121,9 +118,6 @@ class Comment(object): """ comments = ["", self.comment_html_table_th()] - if self._up_up_builds: - logger.debug("get up_up_builds") - comments.extend(self._comment_of_ac(self._up_up_builds[0])) if self._up_builds: comments.extend(self._comment_of_check_item(self._up_builds)) @@ -425,55 +419,30 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") - dd = DistDataset() - dd.set_attr_stime("comment.job.stime") - - # gitee pr tag - gp = GiteeProxy(args.owner, args.repo, args.gitee_token) - gp.delete_tag_of_pr(args.pr, "ci_processing") - jp = JenkinsProxy(args.jenkins_base_url, args.jenkins_user, args.jenkins_api_token) url, build_time, reason = jp.get_job_build_info(os.environ.get("JOB_NAME"), int(os.environ.get("BUILD_ID"))) - dd.set_attr_ctime("comment.job.ctime", build_time) - dd.set_attr("comment.job.link", url) - dd.set_attr("comment.trigger.reason", reason) - - dd.set_attr_stime("comment.build.stime") comment = Comment(args.pr, jp, *args.check_item_comment_files) \ if args.check_item_comment_files else Comment(args.pr, jp) logger.info("comment: build result......") - comment_content = comment.comment_build(gp) - dd.set_attr_etime("comment.build.etime") - dd.set_attr("comment.build.content.html", comment_content) - - if comment.check_build_result() == SUCCESS: - gp.delete_tag_of_pr(args.pr, "ci_failed") - gp.create_tags_of_pr(args.pr, "ci_successful") - dd.set_attr("comment.build.tags", ["ci_successful"]) - dd.set_attr("comment.build.result", "successful") - if args.check_result_file: - comment.comment_compare_package_details(gp, args.check_result_file) + comment_content = comment.comment_build() + success = True + for i in comment_content.split('\n')[2:-1]: + if 'check_build' in i and ':x:FAILED' in i: + success = False + break + body_content = { + "pkg_id": os.getenv('pkg_id'), + "success": success, + "detail": comment_content.replace('\n', '') + } + body = base64.b64encode(json.dumps(body_content).encode('utf-8')).decode() + msg = { + "Header": None, + "Body": body + } + kp = KafkaProducerProxy(brokers=os.getenv('KAFKAURL').split(',')) + if 'test' in os.getenv('service'): + kp.send('software_pkg_ci_checked_test', '', msg) else: - gp.delete_tag_of_pr(args.pr, "ci_successful") - gp.create_tags_of_pr(args.pr, "ci_failed") - dd.set_attr("comment.build.tags", ["ci_failed"]) - dd.set_attr("comment.build.result", "failed") - if args.owner != "openeuler": - comment.get_all_result_to_kafka(url) - - logger.info("comment: at committer......") - comment.comment_at(args.committer, gp) - - dd.set_attr_etime("comment.job.etime") - - # suppress python warning - warnings.filterwarnings("ignore") - logging.getLogger("elasticsearch").setLevel(logging.WARNING) - logging.getLogger("kafka").setLevel(logging.WARNING) - - # upload to es - kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) - query = {"term": {"id": args.comment_id}} - script = {"lang": "painless", "source": "ctx._source.comment = params.comment", "params": dd.to_dict()} - kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) + kp.send('software_pkg_ci_checked', '', msg) diff --git a/src/build/osc_build_k8s.py b/src/build/osc_build_k8s.py index 99d7a46427a197725cf2aa1096013dc0d2956736..651e4764af7faca5cba1196f02e36753aa690887 100755 --- a/src/build/osc_build_k8s.py +++ b/src/build/osc_build_k8s.py @@ -69,15 +69,6 @@ class SinglePackageBuild(object): logger.info("checkout ... ok") - # update package meta file "_service" - self._handle_package_meta(project, work_dir, code_dir) - logger.debug("prepare \"_service\" ... ok") - - # process_service.pl - if not self._prepare_build_environ(project, work_dir): - logger.error("prepare environ ... failed") - return 2 - logger.info("prepare environ ... ok") # osc build @@ -191,40 +182,22 @@ class SinglePackageBuild(object): if self._branch.lower() in Constant.STOPPED_MAINTENANCE_BRANCH: logger.error("branch \"%s\" is no longer maintained!", self._branch) return 1 - if self._branch not in Constant.GITEE_BRANCH_PROJECT_MAPPING: - logger.error("branch \"%s\" not support yet", self._branch) - return 1 has_any_repo_build = False - for project in Constant.GITEE_BRANCH_PROJECT_MAPPING.get(self._branch): - logger.debug("start build project %s", project) - - obs_repos = self.get_need_build_obs_repos(project) - if not obs_repos: - logger.info("all repos ignored of project %s", project) - continue - logger.debug("build obs repos: %s", obs_repos) - - logger.info("project: %s", project) - with open("obs_project", "w") as f: - f.write(project) - - has_any_repo_build = True - ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) - if ret > 0: - logger.debug("build run return %s", ret) - logger.error("build %s %s %s ... %s", project, self._package, self._arch, "failed") - return 1 # finish if any error - else: - logger.info("build %s %s %s ... %s", project, self._package, self._arch, "ok") - break - - # if no repo build, regard as fail - if not has_any_repo_build: - logger.error("package not in any obs projects, please add package into obs") - return 1 - - return 0 + project = 'home:nicliuqi' + obs_repos = [{ + 'repo': 'standard_' + self._arch.replace('-', '_'), + 'mpac': os.getenv('package'), + 'state': 'succeeded*' + }] + ret = self.build_obs_repos(project, obs_repos, spec, work_dir, code_dir) + if ret > 0: + logger.debug("build run return %s", ret) + logger.error("build %s %s %s ... %s", project, self._package, self._arch, "failed") + return 1 # finish if any error + else: + logger.info("build %s %s %s ... %s", project, self._package, self._arch, "ok") + return 0 def init_args(): @@ -259,64 +232,15 @@ if "__main__" == __name__: logging.config.fileConfig(logger_conf_path) logger = logging.getLogger("build") - logger.info("using credential %s", args.account.split(":")[0]) - logger.info("cloning repository https://gitee.com/%s/%s.git ", args.owner, args.repo) - logger.info("clone depth 1") - logger.info("checking out pull request %s", args.pr) + logger.info('osc buils k8s start') from src.utils.dist_dataset import DistDataset from src.proxy.git_proxy import GitProxy from src.proxy.obs_proxy import OBSProxy - from src.proxy.es_proxy import ESProxy from src.proxy.kafka_proxy import KafkaProducerProxy from src.utils.shell_cmd import shell_cmd_live - dd = DistDataset() - dd.set_attr_stime("spb.job.stime") - dd.set_attr("spb.job.link", os.environ["BUILD_URL"]) - dd.set_attr("spb.trigger.reason", os.environ["BUILD_CAUSE"]) - - # suppress python warning - warnings.filterwarnings("ignore") - logging.getLogger("elasticsearch").setLevel(logging.WARNING) - logging.getLogger("kafka").setLevel(logging.WARNING) - - kp = KafkaProducerProxy(brokers=os.environ["KAFKAURL"].split(",")) - - # download repo - dd.set_attr_stime("spb.scm.stime") - gp = GitProxy.init_repository(args.repo, work_dir=args.workspace) - repo_url = "https://{}@gitee.com/{}/{}.git".format(args.account, args.owner, args.repo) - if not gp.fetch_pull_request(repo_url, args.pr, depth=1): - logger.info("fetch finished -") - - dd.set_attr("spb.scm.result", "failed") - dd.set_attr_etime("spb.scm.etime") - dd.set_attr_etime("spb.job.etime") - #dd.set_attr("spb.job.result", "failed") - - # upload to es - query = {"term": {"id": args.comment_id}} - script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), - "params": dd.to_dict()} - kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) - sys.exit(-1) - else: - gp.checkout_to_commit_force("pull/{}/MERGE".format(args.pr)) - logger.info("fetch finished +") - dd.set_attr("spb.scm.result", "successful") - dd.set_attr_etime("spb.scm.etime") - - dd.set_attr_stime("spb.build.stime") spb = SinglePackageBuild(args.package, args.arch, args.branch) rs = spb.build(args.spec, args.workspace, args.code) - dd.set_attr("spb.build.result", "failed" if rs else "successful") - dd.set_attr_etime("spb.build.etime") - - dd.set_attr_etime("spb.job.etime") - - # upload to es - query = {"term": {"id": args.comment_id}} - script = {"lang": "painless", "source": "ctx._source.spb_{}=params.spb".format(args.arch), "params": dd.to_dict()} - kp.send("openeuler_statewall_ci_ac", key=args.comment_id, value=dd.to_dict()) + logger.info('osc build k8s ends') sys.exit(rs) diff --git a/src/lib/comment.sh b/src/lib/comment.sh index 62005d9955e56c4b8e6fce24a361858b43111051..bf7b0a3eddd6debb1cd9c683560ac4ac95ef9a56 100644 --- a/src/lib/comment.sh +++ b/src/lib/comment.sh @@ -71,14 +71,10 @@ function scp_comment_file() { function exec_comment() { log_info "***** Start to exec comment *****" export PYTHONPATH=${shell_path} - python3 ${shell_path}/src/build/gitee_comment.py -o $repo_owner -r $repo -p $prid -c $committer -t ${giteetoken}\ - -b $jenkins_api_host -u $jenkins_user -j $jenkins_api_token -a ${check_item_comment_aarch64} ${check_item_comment_x86}\ - -f ${compare_package_result_x86},${compare_package_result_aarch64} -m ${commentid} + python3 ${shell_path}/src/build/gitee_comment.py -b $jenkins_api_host -u $jenkins_user -j $jenkins_api_token \ + -a ${check_item_comment_aarch64} ${check_item_comment_x86} log_info "***** End to exec comment *****" log_info "***** Start to exec comment to kafka*****" - python3 ${shell_path}/src/build/comment_to_dashboard.py -r $repo -c $committer -m ${commentid} -g $jobtriggertime\ - -k "${prtitle}" -t $prcreatetime -b $tbranch -u $prurl -i $triggerbuildid -p $prid -o $repo_owner \ - --gitee_token ${giteetoken} log_info "***** End to exec comment to kafka*****" } diff --git a/src/lib/main_process.sh b/src/lib/main_process.sh index c4caac7318e676433347d1d7464c004bce3f1b4e..62eefb79ea56b8727316e8d9d79cdee7bf1e23d9 100644 --- a/src/lib/main_process.sh +++ b/src/lib/main_process.sh @@ -184,6 +184,8 @@ function build_packages() { log_info "***** Start to build package *****" comment_file="${repo}_${prid}_${arch}_comment" export PYTHONPATH=${shell_path} + python3 ${SCRIPT_PATCH}/osc_build_k8s.py -o ${repo_owner} -p ${package} -a $arch -c $WORKSPACE -b $tbranch -r ${repo} -m ${commentid} --pr ${prid} -t ${GiteeUserPassword} + python3 ${SCRIPT_PATCH}/extra_work.py checkinstall -a ${arch} -r $tbranch --obs_rpm_host ${obs_rpm_host} --install-root=${WORKSPACE}/install_root/${commentid} -e $WORKSPACE/${comment_file} || echo "continue although run check install failed" for item in $(echo ${package} | sed 's/,/ /g'); do log_info "start build package $item" log_debug "params are [$repo, $branch, $prid, $committer, $arch, $package, $buddy, $WORKSPACE]" @@ -330,7 +332,6 @@ EOF # 执行入口 function main() { - config_osc config_ipv6 config_maven config_gradle @@ -349,6 +350,5 @@ function main() { if [[ $exclusive_arch ]]; then log_info "exclusive_arch not empty" build_packages - compare_package fi } diff --git a/src/utils/download.py b/src/utils/download.py new file mode 100644 index 0000000000000000000000000000000000000000..16198f8a0c453104c85743afadf08f705fb8fa14 --- /dev/null +++ b/src/utils/download.py @@ -0,0 +1,77 @@ +""" +Avoid incomplete downloads due to redirection, use Gitee V5 API to get raw file. +""" +import base64 +import os +import requests +import subprocess +import sys +from urllib.parse import urlparse + + +def repo_branches(owner, repo, access_token): + request_url = 'https://gitee.com/api/v5/repos/{}/{}/branches'.format(owner, repo) + params = { + 'access_token': access_token + } + r = requests.get(request_url, params=params) + if r.status_code != 200: + print('ERROR! Unexpected Error: {}'.format(r.content)) + sys.exit(1) + return [x['name'] for x in r.json()] + + +def repo_file_content(owner, repo, branch, access_token, filepath): + request_url = 'https://gitee.com/api/v5/repos/{}/{}/contents/{}'.format(owner, repo, filepath) + params = { + 'ref': branch, + 'access_token': access_token + } + r = requests.get(request_url, params=params) + if r.status_code == 404: + return '' + elif r.status_code != 200: + print('ERROR! Unexpected Error: {}'.format(r.content)) + sys.exit(1) + return r.json()['content'] + + +def main(): + if len(sys.argv) < 3: + print('ERROR! Lost of required source url and access token.') + sys.exit(1) + url = sys.argv[1] + access_token = sys.argv[2] + urlparse_res = urlparse(url) + if urlparse_res.netloc != 'gitee.com': + cmd = 'wget {}'.format(url) + res = subprocess.call(cmd, shell=True) + sys.exit(res) + urlparse_path = urlparse_res.path + if len(urlparse_path.split('/')) < 6: + print('ERROR! You must apply a file address instead of a repo address.') + sys.exit(1) + _, owner, repo, fmt, branch_filepath = urlparse_path.split('/', 4) + filename = branch_filepath.split('/')[-1] + if fmt != 'raw': + print('ERROR! Source file must be raw format.') + sys.exit(1) + branches = repo_branches(owner, repo, access_token) + for branch in branches: + if branch_filepath.startswith(branch) and len(branch_filepath.split(branch)) == 2: + filepath = branch_filepath.split(branch + '/')[1] + content_b64 = repo_file_content(owner, repo, branch, access_token, filepath) + if not content_b64: + continue + content = base64.b64decode(content_b64) + with open(filename, 'wb') as f: + f.write(content) + print('Download {} successfully.'.format(filename)) + break + if not os.path.exists(filename): + print('Download {} failed.'.format(filename)) + sys.exit(1) + + +if __name__ == '__main__': + main()