diff --git a/tools/oect/src/community/openeuler_community_info_query.py b/tools/oect/src/community/openeuler_community_info_query.py new file mode 100644 index 0000000000000000000000000000000000000000..7ea3cb1faaa18ea6f84f9a689c916c2a8f9d2e6a --- /dev/null +++ b/tools/oect/src/community/openeuler_community_info_query.py @@ -0,0 +1,358 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2021-06-20 +# ******************************************************************************/ +""" +This is a simple script to query that contact person for specific package +""" +import sys +sys.path.append('/home/oect') +import argparse +import os +import re +import shlex +import subprocess +import yaml +import csv +import codecs +from genericpath import isfile +from src.libs.logger import logger +from src.config import constant +from src.libs.base import http, save2csv +from itertools import islice + + +class OpenEulerCommunityRepoInfoQuery(object): + """ + get sig maintainer and contributor of package + """ + def __init__(self): + """ + @description : + ----------- + @param : + ----------- + @returns : + ----------- + """ + + self.signames = [] + self.sig_repo_map = dict() + self.get_local_sigs_yaml() # 读取本地的sigs.yaml内容 + self.exps = dict() + self.get_local_excep_repo_name() + self.openeuler_repo_owner = dict() + self.get_local_openEuler_owmers() + + def get_local_openEuler_owmers(self): + """ + get owner of openeuler packages + Args: + + Returns: + + """ + with open (constant.LOCAL_OPENEULER_OWNERS, 'r',encoding='gbk') as owner_file: + reader = csv.reader(owner_file) + for line in islice(reader, 1, None): + # Package Sig Team Owner Email QA maintainers + self.openeuler_repo_owner[line[0].strip()] = dict() + self.openeuler_repo_owner[line[0].strip()]['Team'] = line[2].strip() + self.openeuler_repo_owner[line[0].strip()]['Owner'] = line[3].strip() + self.openeuler_repo_owner[line[0].strip()]['Email'] = line[4].strip() + self.openeuler_repo_owner[line[0].strip()]['QA'] = line[5].strip() + self.openeuler_repo_owner[line[0].strip()]['maintainers'] = line[6].strip() + + + def get_local_excep_repo_name(self): + """ + get the repo_name:srpm_name spec_name dict + Args: + + Returns: + + """ + with open(constant.LOCAL_REPO_EXCEP, encoding='utf-8') as f: + self.exps = yaml.safe_load(f) + + + def get_local_sigs_yaml(self): + """ + get the content of local sigs.yaml + + returns: + sigs: content of sigs.yaml + exp: + A-Tune:['python-nni', 'python-astor'] + """ + sigs = dict() + sigs_dir = os.listdir(constant.LOCAL_SIGS) + logger.info(sigs_dir) + for sig_dir in sigs_dir: + if sig_dir == 'TC' or isfile(os.path.join(constant.LOCAL_SIGS, sig_dir)): + continue + + self.signames.append(sig_dir) + sigs[sig_dir] = [] + + sig_repos_dir = os.path.join(constant.LOCAL_SIGS, sig_dir, 'src-openeuler') + # community/tree/master/sig/Application/src-openeuler/ + try: + src_oe_pkgs_dir = os.listdir(sig_repos_dir) + except FileNotFoundError as err: + # logger.warning(f"{sig_dir} has no src-openeuler/repo") + continue + + for src_oe_prepos in src_oe_pkgs_dir: + repos_yamls = os.listdir(os.path.join(sig_repos_dir, src_oe_prepos)) + repo_names = [repo_yaml.replace('.yaml', '') for repo_yaml in repos_yamls] + sigs[sig_dir].extend(repo_names) + + self.sig_repo_map = sigs + + + def get_gitee_repo_related_name(self, name_type, name): + """ + get the real package repo name on gitee + + args: + name_type: the name of the package/srpm/spec + name: the value of name + returns: + related names: packsge, srpm, spec + """ + + spec_name = name + gitee_repo_name = name + srpm_name = name + for key, value in self.exps.items(): + if name_type == 'spec': + if value.get("spec") == name: + gitee_repo_name = key + srpm_name = value.get("srpm") + break + if name_type == 'srpm': + if value.get("srpm") == name: + gitee_repo_name = key + spec_name = value.get("spec") + break + if name_type == 'gitee_repo': + if key == name: + spec_name = value.get("spec") + srpm_name = value.get("srpm") + break + return gitee_repo_name, spec_name, srpm_name + + + def query_sig_of_repo(self, package): + """ + get the sig name of specific package + args: + pacakge + returns: + sig_name: SIG name to which package belongs + """ + sig_name = "NA" + if not self.sig_repo_map: + return sig_name + for sig in self.signames: + for repo in self.sig_repo_map[sig]: + if repo == package: + return sig + return sig_name + + + def get_owner_sigs(self, gitee_id): + """ + get the sigs maintained by gitee_id + + args: + gitee_id: Registered gitee ID + returns: + own_sigs: Maintained sig list + """ + own_sigs = [] + for sig_name in self.signames: + sig_owners = self.query_maintainer_info(sig_name) + if gitee_id in sig_owners: + own_sigs.append(sig_name) + + logger.info(f"{gitee_id} maintain {own_sigs}") + return own_sigs + + @staticmethod + def query_maintainer_info(sig_name, info_type='gitee_id'): + """ + get maintainers of specific sig + + args: + sig_name: name of sig + returns: + maintainers: maintainers of sig + """ + + maintainers = [] + sig_maintainer_yaml = constant.LOCAL_SIGS_OWNERS.format(signame = sig_name) + try: + with open(sig_maintainer_yaml, encoding='utf-8') as f: + yaml_content = yaml.safe_load(f) + maintainer_content = yaml_content.get('maintainers', {}) + for maintainer_info in maintainer_content: + maintainers.append(maintainer_info[info_type]) + logger.info(f"{sig_name} 's maintailers: {maintainers}") + except FileNotFoundError as err: + logger.warning(f"{sig_maintainer_yaml}: {err.strerror}") + except KeyError as kerr: + logger.warning(f"{sig_name} has no {kerr} info") + + if maintainers: + return maintainers[0] + else: + return ['NA'] + + @staticmethod + def get_spec_context(package_name, spec_name, branch = 'master'): + """ + get spec file content of package + args: + package: package name/spec filename + returns: + spec_content: content of spec file + """ + + specurl = constant.SPEC_URL.format( + package = package_name, + branch = branch, + specfile = spec_name + ".spec") + logger.info(f"specurl:{specurl}") + resp = http.get(specurl) + + if resp is None or resp.status_code != 200: + logger.warning(f"get spec: None") + return None + spec_str = resp.text + if spec_str is None: + logger.error("spec_str is None") + return None + else: + spec = spec_str.splitlines() + return spec + + def get_latest_contributors(self, package_name, branch, max_eamil_num=1): + """ + get latest contributors's emails + + args: + package: package name/spec filename + max_eamil_num: limit the maximum number of eamils + returns: + emails: eamils of latest contributors + """ + __, spec_name, __ = self.get_gitee_repo_related_name('gitee_repo', package_name) + spec = OpenEulerCommunityRepoInfoQuery.get_spec_context(package_name, spec_name, branch) + if spec is None: + logger.error("get spec of %s failed!", package_name) + return 'NA' + + emails = self.get_emails_of_contributors(spec) + if emails: + return emails[0] + else: + return 'NA' + + @staticmethod + def get_emails_of_contributors(spec, max_eamil_num=1): + """ + analyse the email of contributor in changelog + args: + spec: content of spec file + max_eamil_num: limit the maximum number of eamils + returns: + emails: eamils of latest max_eamil_num contributors + """ + emails = [] + num = 0 + in_changelog = False + for line in spec: + if line.startswith("%changelog"): + in_changelog = True + if in_changelog and line.startswith("*") and num < max_eamil_num: + try: + regular = re.compile(r'[0-9a-zA-Z\.\-]+@[0-9a-zA-Z\.\-]+[com, org]') + email = re.findall(regular, line)[0] + logger.info(f"email: {email}") + emails.append(email) + num = num + 1 + except IndexError as e: + logger.error(f"analyse developer for {line} failed") + emails.append(line) + return emails + + def query_sig_repo(self, sig_name): + """ + query the repo name list of sig_name + Args: + sig_name: name of sig + Returns: + sig_repos: list of repo of sig + """ + + sig_repos = [] + + try: + sig_repos = self.sig_repo_map[sig_name] + except KeyError as ke: + logger.error(ke) + return sig_repos + + def query_full_sig_repos(self): + """ + @description :生成一份gitee上的repo——sig——maintainer信息表 + ----------- + @param : + ----------- + @returns : + ----------- + """ + res_data = [] + for c_sig_name in self.signames: + sig_repos = self.query_sig_repo(c_sig_name) + sig_maintainer = self.query_maintainer_info(c_sig_name) + for repo_name in sig_repos: + res_data.append([repo_name, c_sig_name, sig_maintainer]) + + res_csv_name = "openEuler_full_repos.csv" + save2csv(res_csv_name, res_data, 'w', ["Package", "Sig", "Maintainers"]) + + + def get_related_email(self, repo_name): + """ + @description : 获取软件包的sig组信息、最近的贡献者邮箱信息 + ----------- + @param : + ----------- + @returns : + ----------- + """ + + sig_name = self.query_sig_of_repo(repo_name) + sig_maintainer = self.query_maintainer_info(sig_name) + developer_email = self.get_latest_contributors(repo_name, self.branch) + return sig_name, sig_maintainer, developer_email + + +sig_info_query = OpenEulerCommunityRepoInfoQuery() + +if __name__ == "__main__": + sig_info_query.get_latest_contributors('zip', 'master') \ No newline at end of file diff --git a/tools/oect/src/config/constant.py b/tools/oect/src/config/constant.py new file mode 100644 index 0000000000000000000000000000000000000000..2707e1e084f927425636743dfcb5bf601357a2ca --- /dev/null +++ b/tools/oect/src/config/constant.py @@ -0,0 +1,192 @@ +#!/bin/env python3 +# -*- encoding=utf-8 -*- +""" +# ******************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# [openeuler-jenkins] is licensed under the Mulan PSL v1. +# You can use this software according to the terms and conditions of the Mulan PSL v1. +# You may obtain a copy of Mulan PSL v1 at: +# http://license.coscl.org.cn/MulanPSL +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v1 for more details. +# Author: wangge +# Create: 2021-10-29 +# ******************************************************************** +""" + + +# oect的上层目录 +RUN_DIRECTORY = "/home" + +# 存放配置文件repo的目录 +CONFIG = f"{RUN_DIRECTORY}/oect/src/config" +REPO_PATH = "/etc/yum.repos.d" +BACK_REPO_PATH = "/etc/yum.repos.d/temprepos" +OE_REPO = "huawei.repo" +RT_REPO = "realtime.repo" +ORI_REPO = "openEuler.repo" + +# 当前用户的obs主页 +OBS_OWN_HOME = "https://build.openeuler.org/user/show/senlin" +# OBS 实时日志链接 +OBS_PROJECT_LIVE_LOG= "https://build.openeuler.org/package/live_build_log/{obs_project}/{package}/{repo}/{arch}" +# OBS 账号 +ACCOUNT = "senlin" +# OBS主页 +# OBS_HOST = "https://build.openeuler.org/" +OBS_HOST = "http://117.78.1.88/" +# openeuler dailybuild +DAILYBUILD_URL = "http://121.36.84.172/dailybuild/{branch}/{repo_dir}/Packages" +OPENEULER_REPO_URL = 'https://repo.huaweicloud.com/openeuler/{branch}/{repo_dir}/Packages' + +SIGS_YAML_URL = 'https://gitee.com/openeuler/community/raw/master/sig/sigs.yaml' +SIG_OWNER_URL = 'https://gitee.com/openeuler/community/raw/master/sig/{signame}/OWNERS' +SPEC_URL = 'https://gitee.com/src-openeuler/{package}/raw/{branch}/{specfile}' +LOCAL_REPO_EXCEP = f'{RUN_DIRECTORY}/oect/src/config/pdfs.yaml' +LOCAL_SIGS = '/root/community/sig/' +LOCAL_SIGS_OWNERS = LOCAL_SIGS + '{signame}/sig-info.yaml' +LOCAL_OPENEULER_OWNERS = f'{CONFIG}/openEuler-owner.csv' + +# check_rpm_install_dependence log url +CHECK_RPM_INSTALL_DEPENDENCE = 'https://jenkins.openeuler.org/job/openEuler-OS-build/job/check_rpm_install_dependence_{arch}/{num}/consoleFull' + +# oemaker rpmlist.xml url +OEMAKER_RPMLIST_XML_PATH = 'https://gitee.com/src-openeuler/oemaker/raw/{branch}/rpmlist.xml' + +# openeuler branch bind with obs project +GITEE_BRANCH_PROJECT_MAPPING = { + "master": ["openEuler:Mainline", "openEuler:Epol"], + # "master": ["bringInRely", "openEuler:Extras", "openEuler:Factory", "openEuler:Mainline", "openEuler:Epol"], + "openEuler-20.03-LTS-SP1": ["openEuler:20.03:LTS:SP1", "openEuler:20.03:LTS:SP1:Epol"], + "openEuler-20.03-LTS-SP3": ["openEuler:20.03:LTS:SP3", "openEuler:20.03:LTS:SP3:Epol"], + "openEuler-22.03-LTS-Next": ["openEuler:22.03:LTS:Next", "openEuler:22.03:LTS:Next:Epol"], + "openEuler-22.03-LTS": ["openEuler:22.03:LTS", "openEuler:22.03:LTS:Epol"] +} + +OE_PROJECT_REALTIME_REPO = { + "openEuler:Factory":{ + "name":"Factory", + "baseurl":"http://119.3.219.20:82/openEuler:/Factory/standard_aarch64/", + "enabled":1, + "gpgcheck":0}, + "openEuler:Mainline":{ + "name":"Mainline", + "baseurl":"http://119.3.219.20:82/openEuler:/Mainline/standard_aarch64/", + "enabled":1, + "gpgcheck":0}, + "openEuler:22.03:LTS":{ + "name":"2203_LTS", + "baseurl":"http://119.3.219.20:82/openEuler:/22.03:/LTS/standard_aarch64/", + "enabled":1, + "gpgcheck":0}, + "openEuler:22.03:LTS:Epol":{ + "name":"2203_LTS_Epol", + "baseurl":"http://119.3.219.20:82/openEuler:/22.03:/LTS:/Epol/standard_aarch64/", + "enabled":1, + "gpgcheck":0}, + "openEuler:22.03:LTS:Next":{ + "name":"2203_LTS_Next", + "baseurl":"http://119.3.219.20:82/openEuler:/22.03:/LTS:/Next/standard_aarch64/", + "enabled":1, + "gpgcheck":0}, + "openEuler:22.03:LTS:Next:Epol":{ + "name":"2203_LTS_Next_Epol", + "baseurl":"http://119.3.219.20:82/openEuler:/22.03:/LTS:/Next:/Epol/standard_aarch64/", + "enabled":1, + "gpgcheck":0} +} + +BUILD_REQUIRE_PATTERN = { + 'C': { + 'build_require': {'gcc', 'gcc-c++', 'glibc', 'glibc-common', 'make', 'automake', 'cmake'}, + 'build_command': {'make_build', 'make', 'make ', '%{__make}'} + }, + 'python': { + 'build_require': { + 'python2', 'python-devel', 'python3', 'python2-devel', + 'python3-devel', '/usr/bin/pathfix.py', + 'python3-pandas', 'python3-numpy', 'python3-pyelftools', + '%{required_python}', '%{_py}' + }, + 'build_command': {'py3_build', 'py2_build', '%{__python3}'} + }, + 'java': { + 'build_require': { + 'maven-local', 'mvn', 'ant', 'maven', + 'javapackages-local', 'gradle-local' + }, + 'build_command': { + 'mvn_build', 'mvn ', 'mvn_artifact ', 'gradle ', + 'gradlew ', 'gradle-local', 'gradle_build', 'ant ' + } + }, + 'go': { + 'build_require': { + 'go', 'golang', 'golang-bin', + 'compiler(go-compiler)', + 'go-compilers-golang-compiler' + }, + 'build_command': {'go build', 'gobuild '} + }, + 'erlang': { + 'build_require': {'erlang', 'erlang-crypto'}, + 'build_command': {'erlang_compile'} + }, + 'nodejs': { + 'build_require': {'nodejs', 'nodejs-packaging', 'nodejs-devel'}, + 'build_command': {'nodejs_symlink_deps'} + }, + 'perl': { + 'build_require': { + 'perl-devel', 'perl-generators', + 'perl-interpreter', 'perl' + }, + 'build_command': {'perl'} + }, + 'qt': { + 'build_require': { + 'qt4-devel', 'qt5-qtbase-devel', + 'qt5-qtwebkit-devel', 'qt5-rpm-macros' + }, + 'build_command': {'qmake_qt4', 'qmake_qt5', 'qmake-qt5'} + }, + 'lua': { + 'build_require': {'lua-devel', 'lua'}, + 'build_command': {'LUA_'} + }, + 'ruby': { + 'build_require': { + 'ruby', 'ruby-devel', 'rubygems-devel', + '%{?scl_prefix_ruby}ruby(release)', + '%{?scl_prefix_ruby}rubygems' + }, + 'build_command': {'rake', 'gem build'} + }, + 'rust': { + 'build_require': {'rust-packaging'}, + 'build_command': {'cargo_build'} + }, + 'meson': { + 'build_require': {'meson'}, + 'build_command': {'meson_build', 'meson '} + }, + 'mingw': { + 'build_require': { + 'mingw32-gcc-c++', 'mingw64-gcc-c++', + 'mingw32-filesystem', 'mingw64-filesystem', + 'mingw32-gcc', 'mingw64-gcc' + }, + 'build_command': {'mingw', 'mingw_make '} + }, + 'ocaml': { + 'build_require': {'ocaml'}, + 'build_command': {'jbuilder '} + }, + 'php': { + 'build_require': {'php-devel'}, + 'build_command': {} + } +} + diff --git a/tools/oect/src/libs/logger.py b/tools/oect/src/libs/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..ae85c7678b579bd368a6a594f44f64e7a6cf04e3 --- /dev/null +++ b/tools/oect/src/libs/logger.py @@ -0,0 +1,127 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2020-08-10 +# ******************************************************************************/ + +""" +log module: logger.py +""" + +import logging +import os +import pathlib +from concurrent_log_handler import ConcurrentRotatingFileHandler + +# Configuration file path for data initialization +LOG_PATH = '/var/log/oect' +# Logging level +# The log level option value can only be as follows: DEBUG INFO WARNING ERROR CRITICAL +LOG_CONSOLE_LEVEL = 'DEBUG' +LOG_FILE_LEVEL = 'DEBUG' +# Maximum capacity of each file, the unit is byte, default is 30M +BACKUP_COUNT = 30 +# The size of each log file, in bytes, the default size of a single log file is 30M +MAX_BYTES = 31457280 + +class Logger(object): + """ + operation log of the system + """ + def __init__(self, name=__name__): + self.__current_rotating_file_handler = None + self.__console_handler = None + self.__path = os.path.join(LOG_PATH, "oect.log") + + if not os.path.exists(self.__path): + try: + os.makedirs(os.path.split(self.__path)[0]) + except FileExistsError: + pathlib.Path(self.__path).touch(mode=0o644) + + self.__logger = logging.getLogger(name) + self.__logger.setLevel(LOG_CONSOLE_LEVEL) + + def __init_handler(self): + """ + @description : Initial handler + ----------- + @param :NA + ----------- + @returns :NA + ----------- + """ + + self.__current_rotating_file_handler = ConcurrentRotatingFileHandler( + filename=self.__path, + mode="a", + maxBytes=MAX_BYTES, + backupCount=BACKUP_COUNT, + encoding="utf-8", + use_gzip=True, + ) + self.__console_handler = logging.StreamHandler() + self.__set_formatter() + self.__set_handler() + + def __set_formatter(self): + """ + @description : Set log print format + ----------- + @param :NA + ----------- + @returns :NA + ----------- + """ + formatter = logging.Formatter( + '[%(asctime)s][%(module)s:%(lineno)d|%(funcName)s][%(levelname)s] %(message)s', + datefmt="%a, %d %b %Y %H:%M:%S", + ) + + self.__current_rotating_file_handler.setFormatter(formatter) + self.__console_handler.setFormatter(formatter) + + def __set_handler(self): + self.__current_rotating_file_handler.setLevel(LOG_FILE_LEVEL) + self.__logger.addHandler(self.__current_rotating_file_handler) + self.__logger.addHandler(self.__console_handler) + + @property + def logger(self): + """ + @description :Gets the logger property, both file and console handle + ----------- + @param :NA + ----------- + @returns :NA + ----------- + """ + if not self.__current_rotating_file_handler: + self.__init_handler() + return self.__logger + + @property + def file_handler(self): + """ + @description :Get the file handle to the log + ----------- + @param :NA + ----------- + @returns :NA + ----------- + """ + if not self.__current_rotating_file_handler: + self.__init_handler() + return self.__current_rotating_file_handler + +logger = Logger(__name__).logger \ No newline at end of file