diff --git a/tools/oect/src/community/openeuler_community_info_query.py b/tools/oect/src/community/openeuler_community_info_query.py
index 1ef81447a3f4c2658f731bda5c99ecccc258b261..c745bbd4a1ce486a6decc4d92fd9132f2a9b5023 100644
--- a/tools/oect/src/community/openeuler_community_info_query.py
+++ b/tools/oect/src/community/openeuler_community_info_query.py
@@ -21,12 +21,12 @@ sys.path.append('/home/oect')
import os
import re
import yaml
-import csv
from genericpath import isfile
from src.libs.logger import logger
from src.config import constant
-from src.libs.base import http, save2csv
-from itertools import islice
+from src.libs.base import http
+from src.libs.csvrw import CSVRW
+from src.libs.send_email import SendEmail
class OpenEulerCommunityRepoInfoQuery(object):
@@ -59,17 +59,9 @@ class OpenEulerCommunityRepoInfoQuery(object):
Returns:
"""
- with open (constant.LOCAL_OPENEULER_OWNERS, 'r',encoding='gbk') as owner_file:
- reader = csv.reader(owner_file)
- for line in islice(reader, 1, None):
- # Package Sig Team Owner Email QA maintainers
- self.openeuler_repo_owner[line[0].strip()] = dict()
- self.openeuler_repo_owner[line[0].strip()]['Team'] = line[2].strip()
- self.openeuler_repo_owner[line[0].strip()]['Owner'] = line[3].strip()
- self.openeuler_repo_owner[line[0].strip()]['Email'] = line[4].strip()
- self.openeuler_repo_owner[line[0].strip()]['QA'] = line[5].strip()
- self.openeuler_repo_owner[line[0].strip()]['maintainers'] = line[6].strip()
+ self.openeuler_repo_owner, __ = CSVRW.read_2_dict(constant.LOCAL_OPENEULER_OWNERS, encoding='gbk')
+
def get_local_excep_repo_name(self):
"""
@@ -94,7 +86,7 @@ class OpenEulerCommunityRepoInfoQuery(object):
"""
sigs = dict()
sigs_dir = os.listdir(constant.LOCAL_SIGS)
- logger.info(sigs_dir)
+ # logger.info(sigs_dir)
for sig_dir in sigs_dir:
if sig_dir == 'TC' or isfile(os.path.join(constant.LOCAL_SIGS, sig_dir)):
continue
@@ -321,15 +313,24 @@ class OpenEulerCommunityRepoInfoQuery(object):
@returns :
-----------
"""
- res_data = []
+
+ repo_names = []
+ sig_names = []
+ sig_maintainers = []
+
for c_sig_name in self.signames:
+ # logger.info(f"c_sig_name: {c_sig_name}")
sig_repos = self.query_sig_repo(c_sig_name)
sig_maintainer = self.query_maintainer_info(c_sig_name)
for repo_name in sig_repos:
- res_data.append([repo_name, c_sig_name, sig_maintainer])
+ # logger.info(f"{repo_name}")
+ repo_names.append(repo_name)
+ sig_names.append(c_sig_name)
+ sig_maintainers.append(sig_maintainer)
+ res_data = [repo_names, sig_names, sig_maintainers]
res_csv_name = "openEuler_full_repos.csv"
- save2csv(res_csv_name, res_data, 'w', ["Package", "Sig", "Maintainers"])
+ CSVRW.save_by_column(res_csv_name, res_data, ["Package", "Sig", "Maintainers"])
def get_related_email(self, repo_name):
@@ -348,7 +349,4 @@ class OpenEulerCommunityRepoInfoQuery(object):
return sig_name, sig_maintainer, developer_email
-sig_info_query = OpenEulerCommunityRepoInfoQuery()
-
-if __name__ == "__main__":
- sig_info_query.get_latest_contributors('zip', 'master')
\ No newline at end of file
+sig_info_query = OpenEulerCommunityRepoInfoQuery()
\ No newline at end of file
diff --git a/tools/oect/src/config/global_config.py b/tools/oect/src/config/global_config.py
new file mode 100644
index 0000000000000000000000000000000000000000..dedb351bfa7059c251259b892b6c971e03fa1ece
--- /dev/null
+++ b/tools/oect/src/config/global_config.py
@@ -0,0 +1,37 @@
+#!/usr/bin/python3
+# ******************************************************************************
+# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+# licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ******************************************************************************/
+"""
+Global environment variable value when the tool is running
+"""
+
+import os
+
+# oect top dir
+LIBS_CONFIG_FOLDER = os.path.abspath(os.path.dirname(__file__))
+# path of user-agent.json
+USER_AGENT_JSON = f'{LIBS_CONFIG_FOLDER}/user-agent.json'
+# gitee api config
+GITEE_API_CONFIG = f'{LIBS_CONFIG_FOLDER}/gitee_api_config.yaml'
+# gitee memebers id
+GITEE_OPENEULER_MEMBERS_ID_YAML = f'{LIBS_CONFIG_FOLDER}/oe_memebers_id.yaml'
+
+# OBS 实时日志链接
+DEFAULT_OSCRC_APIURL = "https://build.openeuler.org"
+OBS_PROJECT_LIVE_LOG= DEFAULT_OSCRC_APIURL + "/package/live_build_log/{obs_project}/{package}/{repo}/{arch}"
+
+
+# 抄送人邮箱账号信息
+cc_email='xiasenlin1@huawei.com'
+
+# 收件人邮箱账号信息
+to_email='dev@openeuler.org'
diff --git a/tools/oect/src/libs/base.py b/tools/oect/src/libs/base.py
index 0725f98b813c453d8cbf9963e91bc4571d416d16..bdfcb600197cbbe79c6591a89cdc6450a27f6fea 100644
--- a/tools/oect/src/libs/base.py
+++ b/tools/oect/src/libs/base.py
@@ -16,10 +16,7 @@
import codecs
import csv
-from pdb import line_prefix
-import re
import requests
-import subprocess
from requests.sessions import Session
from retrying import retry
from fake_useragent import UserAgent
@@ -33,18 +30,15 @@ from src.libs.executecmd import ExecuteCmd
def update_yum_repo():
"""
- @description :基于/etc/yum.repo.d目录下的repo配置文件, 执行yum clean all, yum makecache
- -----------
- @param :
- -----------
- @returns :
- -----------
+ yum clean all and yum makecache
+ Args:
+
+ Returns:
+
"""
- yum_clean = ["yum", "clean", "all"]
- yum_makecache = ["yum", "makecache"]
- ExecuteCmd.cmd_status(yum_clean)
- ExecuteCmd.cmd_status(yum_makecache)
+ ExecuteCmd.cmd_status(["yum", "clean", "all"])
+ ExecuteCmd.cmd_status(["yum", "makecache"])
def set_repo(chosed_repo_file, branch):
"""
@@ -57,7 +51,7 @@ def set_repo(chosed_repo_file, branch):
"""
- repo_path = constant.REPO_PATH # dirname of yum repo file
+ repo_path = constant.REPO_PATH
from_repo =f"{global_config.LIBS_CONFIG_FOLDER}/{chosed_repo_file}"
back_repo_path = constant.BACK_REPO_PATH
if not mv_files(repo_path, back_repo_path):
@@ -89,23 +83,6 @@ def set_repo(chosed_repo_file, branch):
update_yum_repo()
return True
-def save2csv(res_name, input_data, save_mode, header, save_encoding = 'utf-8'):
- """
- save inputdata to csv file
- Args:
- res_name: file name
- save_mode: save mode, 'w'
- header:
- input_data: list of data array to save
- Returns:
-
- """
- with codecs.open(res_name, save_mode, save_encoding) as result_file:
- writer = csv.writer(result_file)
- writer.writerow(header)
- for line in input_data:
- writer.writerow(line)
-
class http:
"""
http的相关请求
diff --git a/tools/oect/src/libs/csvrw.py b/tools/oect/src/libs/csvrw.py
new file mode 100644
index 0000000000000000000000000000000000000000..335c59e30af0f7e08822f8edeea68f2d0eb5fe5d
--- /dev/null
+++ b/tools/oect/src/libs/csvrw.py
@@ -0,0 +1,223 @@
+#! /usr/bin/env python
+# coding=utf-8
+# ******************************************************************************
+# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+# licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Author: senlin
+# Create: 2022-06-17
+# ******************************************************************************/
+
+from pickle import TRUE
+import sys
+sys.path.append('/home/oect')
+from src.libs.logger import logger
+import codecs
+import csv
+import pandas as pd
+
+class CSVRW(object):
+ """
+ common useage for reading or writing csv file
+ """
+
+ @classmethod
+ def get_object_type(cls, obj):
+ """
+ query the type of object
+ Args:
+ obj:
+ Returns:
+ obj_type:
+ """
+ obj_type = None
+ if isinstance(obj, tuple):
+ obj_type = "tuple"
+ elif isinstance(obj, dict):
+ obj_type = "dict"
+ elif isinstance(obj, list):
+ obj_type = "list"
+ elif isinstance(obj, set):
+ obj_type = "set"
+
+ return obj_type
+
+ @classmethod
+ def get_keys_of_multi_dict(cls, final_keys, dict_value):
+ """
+ analyse the dict data and return the key's list
+ Args:
+ final_keys: store all of the keys of dict_value
+ dict_value:
+ Returns:
+
+ """
+ if cls.get_object_type(dict_value) != 'dict':
+ return
+ elif cls.get_object_type(dict_value) == 'dict':
+ if not final_keys:
+ last_key = ''
+ else:
+ last_key = final_keys[-1]
+ final_keys.pop()
+
+ for sub_key, sub_value in dict_value.items():
+ if not last_key:
+ final_keys.append(sub_key)
+ else:
+ final_keys.append('_'.join([last_key, sub_key]))
+ cls.get_keys_of_multi_dict(final_keys, sub_value)
+
+ return
+
+ @classmethod
+ def get_line_of_multi_dict(cls, res_list, dict_value):
+ """
+ convert the dict value to list
+ Args:
+ res_list: store the list data
+ dict_value:
+ Returns:
+
+ """
+
+ if cls.get_object_type(dict_value) != 'dict':
+ res_list.append(dict_value)
+ return
+ elif cls.get_object_type(dict_value) == 'dict':
+ for __, value in dict_value.items():
+ cls.get_line_of_multi_dict(res_list, value)
+
+ return
+
+
+ @classmethod
+ def save_by_row(cls, res_name, input_data, csv_title=None, save_mode='w', save_encoding='utf-8'):
+ """
+ save inputdata to csv file
+ Note: If the keys of different elements in the dictionary data to be processed \
+ are different or missing, or the order of keys is chaotic, this function is not applicable.
+ Args:
+ res_name: file name to save
+ input_data: input data to save
+ csv_title: the title of csv file
+ save_mode:
+ save_encoding:
+
+ Returns:
+
+ """
+
+ if cls.get_object_type(input_data) == 'list':
+ if not csv_title:
+ logger.warning(f"we need csv_title for list-type input_data")
+ return False
+
+ with codecs.open(res_name, save_mode, save_encoding) as result_file:
+ writer = csv.writer(result_file)
+ writer.writerow(csv_title)
+ writer.writerows(input_data)
+
+ elif cls.get_object_type(input_data) == 'dict':
+ # Re parse the dictionary key as the csv_title
+ csv_title = []
+ values = list(input_data.values())
+
+ cls.get_keys_of_multi_dict(csv_title, values[0])
+ csv_title.insert(0, 'Main_key')
+ # logger.info(f"analyse csv_title: {csv_title}")
+
+ with codecs.open(res_name, save_mode, save_encoding) as result_file:
+ writer = csv.writer(result_file)
+ writer.writerow(csv_title)
+
+ for main_key, main_value in input_data.items():
+ line = []
+ line.append(main_key)
+ cls.get_line_of_multi_dict(line, main_value)
+ writer.writerow(line)
+ else:
+ logger.warning("Sorry, we do not support other typed input_data")
+ return False
+ logger.info("======Successfully save csv========")
+ return True
+
+ @staticmethod
+ def save_by_column(res_name, input_data, csv_title, save_mode='w', save_encoding='utf-8'):
+ """
+ save dict inputdata to csv file
+ Args:
+ res_name: file name
+ save_mode: save mode
+ csv_title:
+ input_data:
+ Returns:
+
+ """
+ final_data = dict()
+ num_csv_title = len(csv_title)
+ num_data_list = len(input_data)
+ if num_csv_title != num_data_list:
+ logger.warning(f"csv_title({num_csv_title}) does not match data column({num_data_list}): terminate")
+ return
+ for i in range(num_csv_title):
+ final_data[csv_title[i]] = input_data[i]
+ df = pd.DataFrame(final_data, columns=csv_title)
+ df.to_csv(res_name, index=False, mode=save_mode, encoding=save_encoding)
+
+ logger.info("======Successfully save csv========")
+
+
+ @staticmethod
+ def read_2_dict(file, encoding='utf-8'):
+ """
+ read csv data and turn to dict: take csv_title[0] as the main key, every csv_title[x] as second key
+ Note: This function is not recommended if the first column has duplicate values
+
+ Args:
+ file: full path of csv file
+ encode: default encoding parameter for pandas.read_csv
+ Returns:
+ openeuler_repo_owner:
+ exp:
+ openEuler-repos: {
+ 'Sig': 'iSulad',
+ 'Team': 'EulerOS',
+ 'Owner': 'zengweifeng',
+ 'Email': 'zwfeng@huawei.com',
+ 'QA': nan,
+ 'maintainers': "['caihaomin', 'lifeng2221dd1', 'duguhaotian', 'jingxiaolu']"
+ }
+ """
+
+ res_data = dict()
+ if not file.endswith('.csv'):
+ logger.warning(f"{file} is not .csv file")
+
+ try:
+ csv_data = pd.read_csv(file, encoding=encoding)
+ except FileNotFoundError as ferr:
+ logger.error(f"{ferr.strerror}")
+ return None
+
+ # get csv_title of csv
+ csv_title = list(csv_data.head(0))
+ # logger.info(f"This is csv_title: {csv_title}")
+
+ num_of_csv_title = len(csv_title)
+ csv_data_lines = csv_data.values.tolist()
+
+ for line in csv_data_lines:
+ # logger.info(line)
+ main_key = line[0]
+ res_data[main_key] = dict()
+ for idx in range(1, num_of_csv_title):
+ res_data[main_key][csv_title[idx]] = line[idx]
+
+ return res_data, csv_title
diff --git a/tools/oect/src/libs/send_email.py b/tools/oect/src/libs/send_email.py
new file mode 100644
index 0000000000000000000000000000000000000000..a8304e9249ed413bf59e22b60764cd7df173286c
--- /dev/null
+++ b/tools/oect/src/libs/send_email.py
@@ -0,0 +1,149 @@
+#! /usr/bin/env python
+# coding=utf-8
+# ******************************************************************************
+# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved.
+# licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# Author: senlin
+# Create: 2022-02-14
+# ******************************************************************************/
+
+import re
+import csv
+import smtplib
+from email.mime.text import MIMEText
+from email.header import Header
+from src.libs.logger import logger
+from src.config import global_config
+from src.libs.csvrw import CSVRW
+
+# config of sender
+EMAIL='13813374731@163.com'
+PASS='SJSJAISTRUWXFETI'
+SMTP_SERVER = 'smtp.163.com'
+
+
+class SendEmail(object):
+ """
+ email sendor
+ """
+ cc = global_config.cc_email
+ to = global_config.to_email
+ def __init__(self):
+ """
+ @description :
+ -----------
+ @param :
+ -----------
+ @returns :
+ -----------
+ """
+
+
+
+ @classmethod
+ def send_email(cls, message, subject):
+ """
+ send a email
+
+ args:
+ message: Message body
+ returns:
+ True: send email normally
+ False: Abnormal email sending feedback
+ """
+ sender_email = EMAIL
+ sender_email_pass = PASS
+ # cc_emails = self.cc.get('email').split(',')
+ # to_emails = self.to.get('email').split(',')
+ to_emails = []
+ to_emails.extend(cls.cc)
+ to_emails.extend(cls.cc)
+
+
+ msg = MIMEText(message, 'html')
+ msg['Subject'] = Header(subject, "utf-8")
+ msg["From"] = Header(EMAIL)
+ msg["To"] = Header(";".join(cls.cc))
+ msg["Cc"] = Header(";".join(cls.to))
+
+ try:
+ server = smtplib.SMTP_SSL(SMTP_SERVER)
+ server.login(sender_email, sender_email_pass)
+ server.sendmail(sender_email, to_emails, msg.as_string())
+ server.quit()
+ logger.info("send email succeed !")
+ return True
+ except smtplib.SMTPException as err:
+ logger.error(f"send email failed: {err.strerror}!")
+ return False
+
+ @staticmethod
+ def edit_email_content(csv_file, csv_title=None):
+ """
+ edit email content
+ Args:
+ input_dict
+ Returns:
+ message:
+ """
+ # CSVRW.get_object_type()
+ csv_data, csv_title = CSVRW.read_2_dict(csv_file, 'gbk')
+ if not csv_data or not csv_title:
+ logger.warning("Invalid input csv file")
+ return None
+
+ title = """"""
+ for key in csv_title:
+ title += """| %s | """ % (key)
+ title += """
"""
+
+ line = ""
+ for main_key, main_value in csv_data.items():
+ new_line = """"""
+ new_line += """| %s | """ % (main_key)
+
+ for __, sub_value in main_value.items():
+ new_line += """%s | """ % (sub_value)
+ new_line += """
"""
+
+ line = line + new_line
+
+ message = """
+ Hello:
+
+
+ Please solve it as soon as possible.
+ Thanks ~^v^~ !!!
+ """ % (title, line)
+
+ return message
+
+
+ def _check_email(self):
+ """
+ Verify and keep the legal email address
+
+ returns:
+ valid_email: legal email address
+ """
+ valid_email = set()
+ for origin_email in self.to_email:
+ try:
+ regular = re.compile(r'[0-9a-zA-Z\.]+@[0-9a-zA-Z\.]+[com, org]')
+ email = re.findall(regular, origin_email)[0]
+ if email:
+ valid_email.add(email)
+ except IndexError as e:
+ logger.error(f"analyse developer for {email} failed")
+ return list(valid_email)
+