diff --git a/tools/oect/src/community/openeuler_community_info_query.py b/tools/oect/src/community/openeuler_community_info_query.py index 7ea3cb1faaa18ea6f84f9a689c916c2a8f9d2e6a..1ef81447a3f4c2658f731bda5c99ecccc258b261 100644 --- a/tools/oect/src/community/openeuler_community_info_query.py +++ b/tools/oect/src/community/openeuler_community_info_query.py @@ -18,14 +18,10 @@ This is a simple script to query that contact person for specific package """ import sys sys.path.append('/home/oect') -import argparse import os import re -import shlex -import subprocess import yaml import csv -import codecs from genericpath import isfile from src.libs.logger import logger from src.config import constant diff --git a/tools/oect/src/libs/base.py b/tools/oect/src/libs/base.py new file mode 100644 index 0000000000000000000000000000000000000000..0725f98b813c453d8cbf9963e91bc4571d416d16 --- /dev/null +++ b/tools/oect/src/libs/base.py @@ -0,0 +1,205 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2022-03-17 +# ******************************************************************************/ + +import codecs +import csv +from pdb import line_prefix +import re +import requests +import subprocess +from requests.sessions import Session +from retrying import retry +from fake_useragent import UserAgent +from src.config import constant +from src.config import global_config +from src.libs.logger import logger +from src.libs.file_manage import mv_files +from src.libs.file_manage import copy_file +from src.libs.executecmd import ExecuteCmd + + +def update_yum_repo(): + """ + @description :基于/etc/yum.repo.d目录下的repo配置文件, 执行yum clean all, yum makecache + ----------- + @param : + ----------- + @returns : + ----------- + """ + yum_clean = ["yum", "clean", "all"] + yum_makecache = ["yum", "makecache"] + + ExecuteCmd.cmd_status(yum_clean) + ExecuteCmd.cmd_status(yum_makecache) + +def set_repo(chosed_repo_file, branch): + """ + Back up the repo directory and configure a new repo + + Args: + chosed_repo_file: the new repo file for usage + branch: branch related to the chosed_repo_file + Returns: + + """ + + repo_path = constant.REPO_PATH # dirname of yum repo file + from_repo =f"{global_config.LIBS_CONFIG_FOLDER}/{chosed_repo_file}" + back_repo_path = constant.BACK_REPO_PATH + if not mv_files(repo_path, back_repo_path): + logger.error("back repo file to temprepo failed!") + return False + if not copy_file(from_repo, repo_path): + logger.error(f"set the repo cofigfile: {from_repo} failed!") + return False + + if chosed_repo_file == constant.OE_REPO: # no more update + return True + + new_repo_path = f"{repo_path}/{chosed_repo_file}" # 初始repo文件拷贝到repo_path目录下 + projects = constant.GITEE_BRANCH_PROJECT_MAPPING[branch] # 获取 + with open(new_repo_path, "a+") as file: #”w"代表着每次运行都覆盖内容 + for pj in projects: + file.write("\n") + repo = constant.OE_PROJECT_REALTIME_REPO[pj] # 获取映射的project repo URL列表 + file.write("[" + repo['name'] + "]"+"\n") + repo_name = f"name={repo['name']}" + file.write(repo_name + "\n") + repo_baseurl = f"baseurl={repo['baseurl']}" + file.write(repo_baseurl + "\n") + repo_enable = f"enabled={repo['enabled']}" + file.write(repo_enable + "\n") + repo_gpgcheck = f"gpgcheck={repo['gpgcheck']}" + file.write(repo_gpgcheck + "\n") + + update_yum_repo() + return True + +def save2csv(res_name, input_data, save_mode, header, save_encoding = 'utf-8'): + """ + save inputdata to csv file + Args: + res_name: file name + save_mode: save mode, 'w' + header: + input_data: list of data array to save + Returns: + + """ + with codecs.open(res_name, save_mode, save_encoding) as result_file: + writer = csv.writer(result_file) + writer.writerow(header) + for line in input_data: + writer.writerow(line) + +class http: + """ + http的相关请求 + """ + + def __init__(self) -> None: + self.user_agent = UserAgent(path=global_config.USER_AGENT_JSON) + self._request = Session() + + def __enter__(self): + self.set_request(request=self._request) + return self + + def __exit__(self, *args): + self._request.close() + + def set_request(self, request): + """ + set header info + + Args: + request + + Returns: + + """ + + request.headers.setdefault("User-Agent", self.user_agent.random) + return request + + @retry(stop_max_attempt_number=3, stop_max_delay=1500) + def _get(self, url, params=None, **kwargs): + try: + response = self._request.request(method="get", url=url, params=params, **kwargs) + except requests.exceptions.ConnectionError as err: + logger.error(err) + logger.error("Failed to establish a new connection: [Errno 111] Connection refused')") + return None + if response.status_code != 200: + if response.status_code == 410: + logger.warning("Please check the token!") + logger.error(response.text) + raise requests.HTTPError("") + return response + + @retry(stop_max_attempt_number=3, stop_max_delay=1500) + def _post(self, url, data, **kwargs): + response = self._request.request(method="post", url=url, data=data, **kwargs) + if response.status_code not in [200, 201]: + logger.error(response) + raise requests.HTTPError("") + return response + + @classmethod + def get(cls, url, params=None, **kwargs): + """ + get request + + Args: + url: url of post request + params: params of post request + + Returns: + response + + """ + + """http的get请求""" + with cls() as _self: + try: + get_method = getattr(_self, "_get") + response = get_method(url=url, params=params, **kwargs) + except requests.HTTPError: + response = requests.Response() + return response + + @classmethod + def post(cls, url, data, **kwargs): + """ + post request + + Args: + url: url of post request + data: data of post request + + Returns: + response + + """ + + with cls() as _self: + try: + get_method = getattr(_self, "_post") + response = get_method(url=url, data=data, **kwargs) + except requests.HTTPError: + response = requests.Response() + return response diff --git a/tools/oect/src/libs/executecmd.py b/tools/oect/src/libs/executecmd.py new file mode 100644 index 0000000000000000000000000000000000000000..195c1422746d9b341ad53f4467c084c002bdc7a3 --- /dev/null +++ b/tools/oect/src/libs/executecmd.py @@ -0,0 +1,63 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2021-07-3 +# ******************************************************************************/ + +import subprocess +from src.libs.logger import logger + +class ExecuteCmd(object): + """ + Encapsulates the external interface for executing shell statements + """ + @classmethod + def cmd_status(cls, command, time_out=20): + """ + @description :Execute command and return to status + ----------- + @param : + command: command to be executed + ----------- + @returns : + subprocess.run(xxx).returncode + ----------- + """ + try: + ret = subprocess.run(command, shell=False, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, encoding="utf-8", timeout = time_out) + if ret.returncode: + logger.error(f"args:{ret.args} \n stderr:{ret.stderr}") + except subprocess.TimeoutExpired as err: + logger.error(err.stdout) + + return ret.returncode + + @classmethod + def cmd_output(cls, command): + """ + @description : Execute the command and return the output + ----------- + @param : + command: command to be executed + ----------- + @returns : + subprocess.check_output(xxx) + ----------- + """ + try: + subp = subprocess.check_output(command, shell=False, stderr=subprocess.STDOUT, encoding="utf-8") + return subp + except subprocess.CalledProcessError as err: + logger.error(f"{command}:{err}") + return None \ No newline at end of file diff --git a/tools/oect/src/libs/file_manage.py b/tools/oect/src/libs/file_manage.py new file mode 100644 index 0000000000000000000000000000000000000000..f0ec597385fa2c694db5f1428fe6653fac7ff977 --- /dev/null +++ b/tools/oect/src/libs/file_manage.py @@ -0,0 +1,97 @@ +#! /usr/bin/env python +# coding=utf-8 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: senlin +# Create: 2021-07-09 +# ******************************************************************************/ +import sys +sys.path.append('/home/oect') +import os +import hashlib +import shutil +from src.libs.logger import logger + +def get_md5(filename): + """ + @description : get md5 value of file + ----------- + @param : filename + ----------- + @returns : + ----------- + """ + if not os.path.isfile(filename): + logger.error(f"{filename} is not file") + return '00000000' + + md5_hash = hashlib.md5() + with open(filename,'rb') as file: + while True: + data = file.read(4096) + if not data: + break + md5_hash.update(data) + return md5_hash.hexdigest() + +def mv_files(from_path, target_path, file_name = None): + """ + @description : mv files in {from_path} to {target_path} + ----------- + @param : from_path, target_path + ----------- + @returns : + ----------- + """ + def update_file(from_path, target_path, new_file): + target_file_path=os.path.join(target_path, new_file) # 定义文件在目标目录的绝对路径 + # 如果文件在目标文件夹中，则删除该文件 + if os.path.exists(target_file_path): + logger.warning(f"{new_file} already in {target_path}, delete first") + os.remove(target_file_path) + shutil.move(os.path.join(from_path, new_file), target_path) + return True + + if not os.path.isdir(target_path): # 若目标目录不存在，直接创建 + os.mkdir(target_path) + + if file_name is None: # 如果file_name 为None，默认即将移动from_path下的所有文件 + names = os.listdir(from_path) + for i, name in enumerate(names): + update_file(from_path, target_path, name) + else: + update_file(from_path, target_path, file_name) + logger.info(f"move files from {from_path} to {target_path} successfully!") + return True + +def copy_file(from_path, target_path): + """ + from_path: 源文件绝对路径 + target_path: 目的文件夹 + """ + file_name = "" + if os.path.isfile(from_path): + dir_name, file_name = os.path.split(from_path) + target_file_path=os.path.join(target_path, file_name) + + # 判断目的文件夹有同名文件 + if os.path.isfile(target_file_path): + logger.info(f"{file_name} already in {target_path}") + if get_md5(from_path) != get_md5(target_file_path): + shutil.copy(from_path, target_file_path) + else: + if os.path.exists(target_path): + shutil.copy(from_path, target_path) + else: + logger.error(f"{from_path} is not file") + return False + + return True \ No newline at end of file