From 25107c65de87da08f0a923bc37ba7e58dc991abf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E9=87=8D=E9=98=B3?= <15757101689@163.com> Date: Wed, 15 Mar 2023 03:00:10 +0000 Subject: [PATCH 1/2] =?UTF-8?q?update=20tools/accesscontrol-oepkgs-managem?= =?UTF-8?q?ent/doc/=E8=AE=BE=E8=AE=A1=E9=80=BB=E8=BE=91.md.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 汪重阳 <15757101689@163.com> --- ...256\276\350\256\241\351\200\273\350\276\221.md" | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git "a/tools/accesscontrol-oepkgs-management/doc/\350\256\276\350\256\241\351\200\273\350\276\221.md" "b/tools/accesscontrol-oepkgs-management/doc/\350\256\276\350\256\241\351\200\273\350\276\221.md" index 019ba1d..0b9ecd4 100644 --- "a/tools/accesscontrol-oepkgs-management/doc/\350\256\276\350\256\241\351\200\273\350\276\221.md" +++ "b/tools/accesscontrol-oepkgs-management/doc/\350\256\276\350\256\241\351\200\273\350\276\221.md" @@ -1,3 +1,15 @@ + +# 总流程 +1.本地编译测试，安装测试 + +2.上传到gitee仓库中，webhook触发 + +3.进行obs编译构建、安装测试 + +4.测试通过则评论pr + + + ### 设计逻辑 - 部署x86-64和aarch64架构下的k8s集群 @@ -44,4 +56,4 @@ 1.在所属的大工程下，新建一个package，该名字要与gitee上创建的软件包名字一致。 2.创建好之后，点击add file，上传_service文件，上传之前，修改该文件中的url地址为gitee软件包地址。 -- 上传到Gitee组织仓oepkgs \ No newline at end of file +- 上传到Gitee组织仓oepkgs -- Gitee From bf4cdaf886872575ebad2e254ed3ee000dbee58c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B1=AA=E9=87=8D=E9=98=B3?= <15757101689@163.com> Date: Fri, 5 May 2023 01:21:46 +0000 Subject: [PATCH 2/2] update tools/accesscontrol-oepkgs-management/src/git.py. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 汪重阳 <15757101689@163.com> --- .../src/git.py | 171 ++++++++++++------ 1 file changed, 117 insertions(+), 54 deletions(-) diff --git a/tools/accesscontrol-oepkgs-management/src/git.py b/tools/accesscontrol-oepkgs-management/src/git.py index b0ab235..c23cf4c 100644 --- a/tools/accesscontrol-oepkgs-management/src/git.py +++ b/tools/accesscontrol-oepkgs-management/src/git.py @@ -1,14 +1,13 @@ # -*- encoding=utf-8 -*- import os -import logging import re -from cmd import shell_cmd import retrying +from shell_cmd import shell_cmd_live +from shell_cmd import shell_cmd +from build_log import Log - -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger("common") +logger = Log() class GitProxy(object): @@ -34,26 +33,26 @@ class GitProxy(object): repo_dir = os.path.join(work_dir, sub_dir) if work_dir else sub_dir clone_cmd = "cd /var/jenkins_home/; git clone {}".format(repo_url) - ret, _, _ = shell_cmd(clone_cmd) + ret, _, _ = shell_cmd_live(clone_cmd) if not ret: - logger.warning("clone repository failed, %s", ret) + logger.warning("clone repository failed, {}".format(ret)) return 2 return cls(repo_dir) @retrying.retry(retry_on_result=lambda result: result is False, - stop_max_attempt_number=3, wait_fixed=2) + stop_max_attempt_number=3, wait_fixed=2000) def update_repo(self): """ 更新本地仓库 @return: """ update_cmd = "cd /var/jenkins_home/{}; git pull".format(self._repo_dir) - ret, _, _ = shell_cmd(update_cmd) + ret, _, _ = shell_cmd_live(update_cmd) if ret: - logger.warning("update repository failed, %s", ret) - return 2 + logger.warning("update repository failed, {}".format(ret)) + return False return True def fetch_pull_request(self, url, pr_num): @@ -63,11 +62,11 @@ class GitProxy(object): :param pr_num: pr编号 :return: """ - fetch_cmd = "cd /var/jenkins_home/{}; git fetch --depth {} {} +refs/pull/{}/MERGE:refs/pull/{}/MERGE".format( - self._repo_dir, 4, url, pr_num, pr_num) - ret, out, _ = shell_cmd(fetch_cmd, cap_out=True, cmd_verbose=False) + fetch_cmd = "cd /var/jenkins_home/{}; git fetch {} pull/{}/head:pr_{}".format( + self._repo_dir, url, pr_num, pr_num) + ret, out, _ = shell_cmd_live(fetch_cmd, cap_out=True, cmd_verbose=False) if ret: - logger.error("git fetch failed,%s ,%s", ret, out) + logger.error("Git fetch failed,{} {}".format(ret, out)) return False return True @@ -78,11 +77,11 @@ class GitProxy(object): :param file_path: 文件完整路径 sig/hpc/src-oepkgs/m/mopac.yaml :return: StringIO """ - get_content_cmd = "cd /var/jenkins_home/{}; git show refs/pull/{}/MERGE:{}".format( - self._repo_dir, pr_num, file_path) - ret, out, _ = shell_cmd(get_content_cmd, cap_out=True) + get_content_cmd = "cd /var/jenkins_home/{}; git show pr_{}:{}".format( + self._repo_dir, pr_num, file_path) + ret, out, _ = shell_cmd_live(get_content_cmd, cap_out=True) if ret: - logger.warning("get file content of commit failed, %s", ret) + logger.error("Get file content of commit failed, {}".format(ret)) return 2 return out @@ -93,139 +92,203 @@ class GitProxy(object): :param pr_num: refs/pull/{pr_num}/MERGE :return: list<string> """ - diff_files_cmd = "cd /var/jenkins_home/{}; git diff --name-only refs/pull/{}/MERGE".format( + diff_files_cmd = "cd /var/jenkins_home/{}; git diff master --name-only --diff-filter=ACM pr_{}".format( self._repo_dir, pr_num) - ret, out, _ = shell_cmd(diff_files_cmd, cap_out=True) + ret, out, _ = shell_cmd_live(diff_files_cmd, cap_out=True) if ret: - logger.error("get diff files of commits failed, %s", ret) + logger.error("Get diff files of commits failed, {}!".format(ret)) return [] return out + @retrying.retry(retry_on_exception=lambda exception: isinstance(exception, IndexError), + stop_max_attempt_number=3, wait_fixed=2000) def check_file_path(self, pr_num): """ 验证路径 :param pr_num: """ - logger.info("Start check repo path is true ?") + logger.info("Check the repo file name and path are correct ?") files_path = self.diff_files_between_commits(pr_num) for file_path in files_path: + if "sig-info.yaml" in file_path: + continue file_list = file_path.split('/') - if file_list[0].strip() != "sig".strip() or file_list[2].strip() != "src-oepkgs".strip(): - logger.error("file_list path error") + logger.error("Path error, first dir not sig or third dir not src-oepkgs !") return 2 elif file_list[3].strip().lower() != file_list[4].strip()[0].lower(): - logger.error("{} last path error".format(file_list[-1])) + logger.error( + "{} repo name and dir name not same, please change it !".format(file_list[-1])) return 2 elif file_list[4].split(".")[-1] != "yaml".strip(): - logger.error("yaml name error") + logger.error("{} Yaml name error, without .yaml please add !".format(file_list)) return 2 logger.info("----END CHECK----") return 0 + @retrying.retry(retry_on_exception=lambda exception: isinstance(exception, IndexError), + stop_max_attempt_number=3, wait_fixed=2000) def check_repo_num(self, pr_num): """ 验证此次建仓库数量是否小于100 @param pr_num: pr序数 @return: """ - logger.info("Start check files number is more than 100 ?") + logger.info("Check whether the number of files is greater than 100 ?") files_path = self.diff_files_between_commits(pr_num) if len(files_path) > 101: - logger.error("create repo more 100, please delete some") + logger.error("Create repo more 100, please delete some !") return 2 logger.info("----END CHECK----") return 0 + @retrying.retry(retry_on_exception=lambda exception: isinstance(exception, IndexError), + stop_max_attempt_number=3, wait_fixed=2000) def check_invalid_file(self, pr_num, invalid_file=".keep"): """ check invalid file """ - logger.info("Start check files have invalid file ?") + logger.info("Check for invalid files ?") filenames = self.diff_files_between_commits(pr_num) for file_name in filenames: if invalid_file in file_name: - logger.error("'.keep' is invalid file, please delete") + logger.error("'.keep' is invalid file, please delete !") return 2 logger.info("----END CHECK----") return 0 + @retrying.retry(retry_on_exception=lambda exception: isinstance(exception, IndexError), + stop_max_attempt_number=3, wait_fixed=2000) def check_name_isdigit(self, pr_num): """ check name not start with a number @param pr_num: @return: """ - logger.info("Start check is the repo name complies with specifications ?") + logger.info("Check whether the repo name conforms to the specification ?") files_path = self.diff_files_between_commits(pr_num) for file_path in files_path: yaml_name = file_path.split('/')[-1] if yaml_name == "sig-info.yaml" or "yaml" not in yaml_name: continue + change_type = self.check_a_m_d(yaml_name, pr_num) + if change_type == "M": + continue + elif change_type == "D": + logger.error("Can not delete yaml files") + return 2 f = self.get_content_of_file_with_commit(file_path, pr_num) f = f[0] repo_name = f.split(":")[-1].strip() result = re.search("\A(?!_)(?!.*?_$)(?!-)(?!.*-$)(?!\+)([^.])[-a-zA-Z0-9_.\u4e00-\u9fa5\uff00-\uffff+ ]+\Z", repo_name) if not result: - logger.error("repo_name: {} has an error".format(repo_name)) + logger.error( + "{} is not complies with specifications, please change it !".format(repo_name)) return 2 logger.info("----END CHECK----") return 0 + def check_a_m_d(self, yaml_name, pr_num): + analyzed_cmd = "cd /var/jenkins_home/{}; git log pr_{} -1 --pretty=format: --name-status --no-merges" \ + "| grep {} | awk -F' ' '{{print $1}}' | cat".format(self._repo_dir, pr_num, + yaml_name) + out = shell_cmd(analyzed_cmd) + return out + + @retrying.retry(retry_on_exception=lambda exception: isinstance(exception, IndexError), + stop_max_attempt_number=3, wait_fixed=2000) def check_name_is_exist(self, pr_num): """ check name is exist? @param pr_num: @return: """ - logger.info("start check name is exist in repo ?") + logger.info("Check whether the repo already exists ?") files_path = self.diff_files_between_commits(pr_num) for file_path in files_path: yaml_name = file_path.split('/')[-1] if yaml_name == "sig-info.yaml" or "yaml" not in yaml_name: continue - get_files_num_cmd = "cd /var/jenkins_home/{}; git ls-files | grep {}".format(self._repo_dir, yaml_name) +\ - "| awk -F '/' '{print $NF}'" + "| grep ^{}$".format(yaml_name) - ret, out, _ = shell_cmd(get_files_num_cmd, cap_out=True) + change_type = self.check_a_m_d(yaml_name, pr_num) + if change_type == "M": + continue + elif change_type == "D": + logger.error("Can not delete yaml files") + return 2 + get_files_num_cmd = "cd /var/jenkins_home/{}; git ls-files | grep -i {}".format(self._repo_dir, yaml_name) \ + + "| awk -F '/' '{print $NF}'" + "| grep -i ^{}$".format(yaml_name) + ret, out, _ = shell_cmd_live(get_files_num_cmd, cap_out=True) if not ret: - logger.error("{} is exist".format(yaml_name)) + logger.error("{} is exist, please find in repo !".format(yaml_name)) return 2 logger.info("----END CHECK----") return 0 - @retrying.retry(retry_on_result=lambda result: result is IndexError, - stop_max_attempt_number=3, wait_fixed=2) - def compare_yaml_repo_name(self, url, pr_num, depth=4, progress=False): + @retrying.retry(retry_on_exception=lambda exception: isinstance(exception, IndexError), + stop_max_attempt_number=3, wait_fixed=2000) + def compare_yaml_repo_name(self, pr_num): """ - check name is exist? + check name @param pr_num: - @param url: - @param depth - @param progress @return: """ - logger.info("Start compare yaml name and repo name is same ?") + logger.info("Check whether yaml name and repo name are the same ?") files_path = self.diff_files_between_commits(pr_num) for file_path in files_path: yaml_name = file_path.split('/')[-1] - if yaml_name == "sig-info.yaml" or "yaml" not in yaml_name: + if "yaml" not in yaml_name: + continue + if yaml_name == "sig-info.yaml": + dir_name = file_path.split('/')[-2] + f = self.get_content_of_file_with_commit(file_path, pr_num) + f = f[0] + if f == 2: + return 2 + repo_name = f.split(":")[-1] + logger.info(yaml_name) + if repo_name.strip() != dir_name: + logger.error("sig-info.yaml [name] {} and dir [name] {} not same".format(repo_name, dir_name)) + return 2 continue - fetch_cmd = "cd /var/jenkins_home/{}; git fetch {} --depth {} {} +refs/pull/{}/MERGE:refs/pull/{}/MERGE".\ - format(self._repo_dir, "--progress" if progress else "", depth, url, pr_num, pr_num) - ret, out, _ = shell_cmd(fetch_cmd, cap_out=True, cmd_verbose=False) - if ret: - logger.error("git fetch failed,%s ,%s", ret, out) + change_type = self.check_a_m_d(yaml_name, pr_num) + if change_type == "D": + logger.error("Can not delete yaml files") return 2 f = self.get_content_of_file_with_commit(file_path, pr_num) f = f[0] + if f == 2: + return 2 repo_name = f.split(":")[-1] yaml_name = yaml_name.split('.yaml')[0] logger.info(yaml_name) if repo_name.strip() != yaml_name.strip(): - logger.error("repo_name: {} and yaml_name: {} name not same, please change".format(repo_name, - yaml_name)) + logger.error( + "repo_name: {} and yaml_name: {} name not same, please change !".format(repo_name, + yaml_name)) return 2 + sig_info_yaml_check_res = self.check_siginfo(pr_num) + if sig_info_yaml_check_res == 2: + return 2 logger.info("----END CHECK----") return 0 + + def check_siginfo(self, pr_num): + """ + check sig-info.yaml change + """ + logger.info("Check sig-info.yaml ?") + files_path = self.diff_files_between_commits(pr_num) + for file_path in files_path: + yaml_name = file_path.split('/')[-1] + change_type = self.check_a_m_d(yaml_name, pr_num) + if yaml_name == "sig-info.yaml" and change_type == "M": + dir_name = file_path.split('/')[-2] + f = self.get_content_of_file_with_commit(file_path, pr_num) + f = f[0] + repo_name = f.split(":")[-1] + if repo_name.strip() != dir_name.strip(): + logger.error("You can not change sig-info.yaml [name] {} or [description]".format(dir_name)) + return 2 + return 0 -- Gitee