From 2e90670b54f11cf64e3a2bc487d35c24d15190de Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Tue, 1 Dec 2020 16:08:50 +0800 Subject: [PATCH 01/64] Add gitignore for python Setup basic gitignore for python development. Python3 will be used for this project, as well as some shell script. --- dispatch/.gitignore | 42 ++++++++++++++++++++++++++++++++++++++++++ generate/.gitignore | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 dispatch/.gitignore create mode 100644 generate/.gitignore diff --git a/dispatch/.gitignore b/dispatch/.gitignore new file mode 100644 index 0000000..41e5e3a --- /dev/null +++ b/dispatch/.gitignore @@ -0,0 +1,42 @@ +__pycache__/ +*.py[cod] +*$py.class + +*.so + +*.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +*.manifest +*.spec + +pip-log.txt +pip-delete-this-directory.txt + +docs/_build/ + +_pypackages/ + +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ diff --git a/generate/.gitignore b/generate/.gitignore new file mode 100644 index 0000000..41e5e3a --- /dev/null +++ b/generate/.gitignore @@ -0,0 +1,42 @@ +__pycache__/ +*.py[cod] +*$py.class + +*.so + +*.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +*.manifest +*.spec + +pip-log.txt +pip-delete-this-directory.txt + +docs/_build/ + +_pypackages/ + +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ -- Gitee From 8ba075959e8e9e51fae1d8be619ad12a49f44046 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 10:23:55 +0800 Subject: [PATCH 02/64] add: gen_report.py, try to analyse logs Starting to try access log path according to date --- generate/gen_report.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 generate/gen_report.py diff --git a/generate/gen_report.py b/generate/gen_report.py new file mode 100644 index 0000000..827e6b3 --- /dev/null +++ b/generate/gen_report.py @@ -0,0 +1,36 @@ +#!/usr/bin/python3 +# coding: utf-8 + +import os +import time +import logging + +def begin() -> Boolean, str, str, str: + # generate local date in format 'YYYYMMDD', e.g. '20201202' + local_date = time.strftime('%Y%m%d', time.localtime(time.time())) + # find log and output path for today + log_path = f'/result/autotest/actuators/{local_date}/log' + output_path = f'/result/autotest/actuators/{local_date}/output' + has_build_today = True + if not os.path.exists(log_path): + logging.log(logging.WARNING, f'未检测到{local_date}的构建日志目录') + has_build_today = False + if not os.path.exists(output_path): + logging.log(logging.WARNING, f'未检测到{local_date}的构建输出目录') + has_build_today = False + return has_build_today, local_date, log_path, output_path + + +def analyse(local_date: str, log_path: str, output_path: str): + log_files = [] + for home, dirs, files in os.walk(log_path): + for filename in files: + log_files.append(os.path.join(home, filename)) + for file_path in log_files: + print(file_path) + +if __name__ == '__main__: + has_build, local_date, log_path, output_path = begin() + if not has_build: + return + analyse(local_date, log_path, output_path) -- Gitee From 38d864cfa21a201007ed75e4a0e7e45ff5321b95 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 10:28:52 +0800 Subject: [PATCH 03/64] fix: correct typing Correct typing for multi-value return - should be Tuple --- generate/gen_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 827e6b3..4f1386f 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -5,7 +5,7 @@ import os import time import logging -def begin() -> Boolean, str, str, str: +def begin() -> Tuple[bool, str, str, str]: # generate local date in format 'YYYYMMDD', e.g. '20201202' local_date = time.strftime('%Y%m%d', time.localtime(time.time())) # find log and output path for today -- Gitee From 14bd0c7719ecaf31689cd02e2d9f4e4f96912943 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 10:30:04 +0800 Subject: [PATCH 04/64] fix: typo --- generate/gen_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 4f1386f..b6f2a28 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -29,7 +29,7 @@ def analyse(local_date: str, log_path: str, output_path: str): for file_path in log_files: print(file_path) -if __name__ == '__main__: +if __name__ == '__main__': has_build, local_date, log_path, output_path = begin() if not has_build: return -- Gitee From 13081dbe83b89cfddcc8737d4abe59c32735134e Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 10:32:55 +0800 Subject: [PATCH 05/64] add: use sys.exit when no build for today occur --- generate/gen_report.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index b6f2a28..46f20be 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -2,6 +2,7 @@ # coding: utf-8 import os +import sys import time import logging @@ -32,5 +33,6 @@ def analyse(local_date: str, log_path: str, output_path: str): if __name__ == '__main__': has_build, local_date, log_path, output_path = begin() if not has_build: - return + # exit no. 2: no build today + sys.exit(2) analyse(local_date, log_path, output_path) -- Gitee From 05ea84a476f8b482e39b1fafa2a0aa17b19a5c98 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 11:14:20 +0800 Subject: [PATCH 06/64] add: gitignore for idea --- dispatch/.gitignore | 2 ++ generate/.gitignore | 2 ++ generate/gen_report.py | 5 +++-- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/dispatch/.gitignore b/dispatch/.gitignore index 41e5e3a..440357d 100644 --- a/dispatch/.gitignore +++ b/dispatch/.gitignore @@ -40,3 +40,5 @@ venv/ ENV/ env.bak/ venv.bak/ + +.idea/ diff --git a/generate/.gitignore b/generate/.gitignore index 41e5e3a..440357d 100644 --- a/generate/.gitignore +++ b/generate/.gitignore @@ -40,3 +40,5 @@ venv/ ENV/ env.bak/ venv.bak/ + +.idea/ diff --git a/generate/gen_report.py b/generate/gen_report.py index 46f20be..40d06c1 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -5,6 +5,7 @@ import os import sys import time import logging +from typing import Tuple def begin() -> Tuple[bool, str, str, str]: # generate local date in format 'YYYYMMDD', e.g. '20201202' @@ -27,8 +28,8 @@ def analyse(local_date: str, log_path: str, output_path: str): for home, dirs, files in os.walk(log_path): for filename in files: log_files.append(os.path.join(home, filename)) - for file_path in log_files: - print(file_path) + main_logs = [log_file for log_file in log_files if log_file.startswith(f'build_{local_date}')] + if __name__ == '__main__': has_build, local_date, log_path, output_path = begin() -- Gitee From 9aacc4e5121fcf4ef18587cdb1654e7991cfb7c6 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 11:16:37 +0800 Subject: [PATCH 07/64] Fix: move gitignore file path --- dispatch/.gitignore => .gitignore | 0 generate/.gitignore | 44 ------------------------------- 2 files changed, 44 deletions(-) rename dispatch/.gitignore => .gitignore (100%) delete mode 100644 generate/.gitignore diff --git a/dispatch/.gitignore b/.gitignore similarity index 100% rename from dispatch/.gitignore rename to .gitignore diff --git a/generate/.gitignore b/generate/.gitignore deleted file mode 100644 index 440357d..0000000 --- a/generate/.gitignore +++ /dev/null @@ -1,44 +0,0 @@ -__pycache__/ -*.py[cod] -*$py.class - -*.so - -*.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -*.manifest -*.spec - -pip-log.txt -pip-delete-this-directory.txt - -docs/_build/ - -_pypackages/ - -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -.idea/ -- Gitee From 00529e340fbe3a9f4211c31873fc0aa72092bd9b Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 11:23:54 +0800 Subject: [PATCH 08/64] Add: filter main task log and subtask log --- generate/gen_report.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 40d06c1..53d4dc4 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -5,7 +5,8 @@ import os import sys import time import logging -from typing import Tuple +from typing import Tuple, List + def begin() -> Tuple[bool, str, str, str]: # generate local date in format 'YYYYMMDD', e.g. '20201202' @@ -21,19 +22,22 @@ def begin() -> Tuple[bool, str, str, str]: logging.log(logging.WARNING, f'未检测到{local_date}的构建输出目录') has_build_today = False return has_build_today, local_date, log_path, output_path - -def analyse(local_date: str, log_path: str, output_path: str): + +def find_log_files(local_date: str, log_path: str) -> Tuple[List[str], List[str]]: log_files = [] for home, dirs, files in os.walk(log_path): for filename in files: log_files.append(os.path.join(home, filename)) - main_logs = [log_file for log_file in log_files if log_file.startswith(f'build_{local_date}')] - + main_logs = [log_file for log_file in log_files if log_file.startswith( + f'build_{local_date}')] + subtask_logs = list(set(log_files) - set(main_logs)) + return main_logs, subtask_logs + if __name__ == '__main__': has_build, local_date, log_path, output_path = begin() if not has_build: - # exit no. 2: no build today - sys.exit(2) - analyse(local_date, log_path, output_path) + # exit no. 2: no build today + sys.exit(2) + main_logs, subtask_logs = find_log_files(local_date, log_path) -- Gitee From 2562cce5f8d264515004cee5577b34d4057063f9 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 12:00:59 +0800 Subject: [PATCH 09/64] Add: read main log files info --- generate/gen_report.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 53d4dc4..d156665 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -5,7 +5,7 @@ import os import sys import time import logging -from typing import Tuple, List +from typing import Tuple, List, Dict def begin() -> Tuple[bool, str, str, str]: @@ -35,9 +35,31 @@ def find_log_files(local_date: str, log_path: str) -> Tuple[List[str], List[str] return main_logs, subtask_logs +def read_main_info(main_logs: List[str]) -> Dict[str, List[str]]: + solution_path_dict = {} + for log_file in main_logs: + with open(log_file, 'r') as fp: + lines = fp.readlines() + + solution = "" + for line in lines: + if 'Solution:' in line: + solution = line.split()[3] + break + + if not solution: + # cannot detect solution, call it unknown + solution = "Unknown" + + solution_path_dict[solution] = solution_path_dict.get(solution, []).append('/'.join(log_file.split('/')[:-1])) + return solution_path_dict + + if __name__ == '__main__': has_build, local_date, log_path, output_path = begin() if not has_build: # exit no. 2: no build today sys.exit(2) main_logs, subtask_logs = find_log_files(local_date, log_path) + solution_path_dict = read_main_info(main_logs) + print(solution_path_dict) -- Gitee From dbb160811dee4cf3364884ede04655ad10b769cb Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 16:19:02 +0800 Subject: [PATCH 10/64] Add: read info from subtask logs --- generate/gen_report.py | 65 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index d156665..46b43ba 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -1,11 +1,11 @@ #!/usr/bin/python3 # coding: utf-8 +import logging import os import sys import time -import logging -from typing import Tuple, List, Dict +from typing import Dict, List, Tuple def begin() -> Tuple[bool, str, str, str]: @@ -24,42 +24,91 @@ def begin() -> Tuple[bool, str, str, str]: return has_build_today, local_date, log_path, output_path -def find_log_files(local_date: str, log_path: str) -> Tuple[List[str], List[str]]: +def find_log_files( + local_date: str, log_path: str) -> Tuple[List[str], List[str]]: log_files = [] + # get all log files for home, dirs, files in os.walk(log_path): for filename in files: log_files.append(os.path.join(home, filename)) + # seperate main task log and subtask log main_logs = [log_file for log_file in log_files if log_file.startswith( f'build_{local_date}')] subtask_logs = list(set(log_files) - set(main_logs)) return main_logs, subtask_logs -def read_main_info(main_logs: List[str]) -> Dict[str, List[str]]: +def read_main_log(main_logs: List[str]) -> Dict[Tuple[str, str], List[str]]: solution_path_dict = {} for log_file in main_logs: with open(log_file, 'r') as fp: lines = fp.readlines() solution = "" + producer = "" for line in lines: if 'Solution:' in line: solution = line.split()[3] - break + if 'Producer:' in line: + producer = line.split()[3] if not solution: # cannot detect solution, call it unknown solution = "Unknown" - solution_path_dict[solution] = solution_path_dict.get(solution, []).append('/'.join(log_file.split('/')[:-1])) + if not producer: + # cannot detect producer, call it unknown + producer = "Unknown" + + solution_path_dict[(solution, producer)] = os.path.dirname(log_file) return solution_path_dict +def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ + Tuple, List[str]]) -> Tuple[Dict[Tuple, str], Dict[Tuple, str]]: + build_status_dict = {} + build_output_dict = {} + for solution_producer, path in solution_path_dict.items(): + build_status_dict[solution_producer] = [] + build_output_dict[solution_producer] = [] + for subtask_log in subtask_logs: + if subtask_log.startswith(path): + with open(subtask_log, 'r') as fp: + lines = fp.readlines() + for line in lines: + if "Build subtask image name:" in line: + image_name = line.split()[-1].split(':')[0] + break + for line in lines[::-1]: + if "Build subtask output dir:" in line: + subtask_output = line.split()[-1] + if "Build subtask result:" in line: + subtask_result = line.split()[-1] + if subtask_result == 'FAIL': + subtask_output = "Not Available" + break + build_status_dict[solution_producer].append( + f"{image_name}: f{subtask_result}") + build_output_dict[solution_producer].append( + f"{image_name}: f{subtask_output}") + return build_status_dict, build_output_dict + + if __name__ == '__main__': has_build, local_date, log_path, output_path = begin() if not has_build: # exit no. 2: no build today sys.exit(2) main_logs, subtask_logs = find_log_files(local_date, log_path) - solution_path_dict = read_main_info(main_logs) - print(solution_path_dict) + solutions_paths = read_main_log(main_logs) + build_status, build_output = read_subtask_log( + subtask_logs, solutions_paths) + for solution_producer in build_status.keys(): + print('===================') + solution, producer = solution_producer[0], solution_producer[1] + print(f'Solution: {solution} Producer: {producer}') + status_list = build_status.get(solution_producer) + output_list = build_output.get(solution_producer) + for status, output in zip(build_status, build_output): + print(status) + print(output) -- Gitee From 93e8d9adf331db4ce1a6c7290aaea9b51a9bc56c Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 16:23:33 +0800 Subject: [PATCH 11/64] Fix: empty output due to wrong main log detect --- generate/gen_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 46b43ba..bfc338b 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -32,8 +32,8 @@ def find_log_files( for filename in files: log_files.append(os.path.join(home, filename)) # seperate main task log and subtask log - main_logs = [log_file for log_file in log_files if log_file.startswith( - f'build_{local_date}')] + main_logs = [ + log_file for log_file in log_files if f'build_{local_date}' in log_file] subtask_logs = list(set(log_files) - set(main_logs)) return main_logs, subtask_logs -- Gitee From baf8d94f9a3ff9357602cf84157e255438825c97 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 2 Dec 2020 16:49:39 +0800 Subject: [PATCH 12/64] fix: typing correction, add some comments --- generate/gen_report.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index bfc338b..ca7da61 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -11,10 +11,11 @@ from typing import Dict, List, Tuple def begin() -> Tuple[bool, str, str, str]: # generate local date in format 'YYYYMMDD', e.g. '20201202' local_date = time.strftime('%Y%m%d', time.localtime(time.time())) - # find log and output path for today + # construct log and output path for today log_path = f'/result/autotest/actuators/{local_date}/log' output_path = f'/result/autotest/actuators/{local_date}/output' has_build_today = True + # check if there is build today by check log/output path if not os.path.exists(log_path): logging.log(logging.WARNING, f'未检测到{local_date}的构建日志目录') has_build_today = False @@ -31,19 +32,19 @@ def find_log_files( for home, dirs, files in os.walk(log_path): for filename in files: log_files.append(os.path.join(home, filename)) - # seperate main task log and subtask log - main_logs = [ - log_file for log_file in log_files if f'build_{local_date}' in log_file] + # separate main task log and subtask log + main_logs = [log for log in log_files if f'build_{local_date}' in log] subtask_logs = list(set(log_files) - set(main_logs)) return main_logs, subtask_logs -def read_main_log(main_logs: List[str]) -> Dict[Tuple[str, str], List[str]]: +def read_main_log(main_logs: List[str]) -> Dict[tuple, str]: solution_path_dict = {} for log_file in main_logs: with open(log_file, 'r') as fp: lines = fp.readlines() + # find solution and producer solution = "" producer = "" for line in lines: @@ -55,30 +56,35 @@ def read_main_log(main_logs: List[str]) -> Dict[Tuple[str, str], List[str]]: if not solution: # cannot detect solution, call it unknown solution = "Unknown" - if not producer: # cannot detect producer, call it unknown producer = "Unknown" + # record path of the main log - subtask logs will be in here solution_path_dict[(solution, producer)] = os.path.dirname(log_file) return solution_path_dict def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ - Tuple, List[str]]) -> Tuple[Dict[Tuple, str], Dict[Tuple, str]]: + tuple, str]) -> Tuple[Dict[tuple, List[str]], Dict[tuple, List[str]]]: build_status_dict = {} build_output_dict = {} + # check subtask logs by solution and producer for solution_producer, path in solution_path_dict.items(): build_status_dict[solution_producer] = [] build_output_dict[solution_producer] = [] for subtask_log in subtask_logs: if subtask_log.startswith(path): + # this subtask log belongs to this solution/producer with open(subtask_log, 'r') as fp: lines = fp.readlines() + image_name = 'unknown' for line in lines: if "Build subtask image name:" in line: image_name = line.split()[-1].split(':')[0] break + subtask_result = 'UNKNOWN' + subtask_output = 'unknown' for line in lines[::-1]: if "Build subtask output dir:" in line: subtask_output = line.split()[-1] @@ -87,28 +93,36 @@ def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ if subtask_result == 'FAIL': subtask_output = "Not Available" break + # record status and output dir, the lists have same length build_status_dict[solution_producer].append( - f"{image_name}: f{subtask_result}") + f"{image_name}: {subtask_result}") build_output_dict[solution_producer].append( - f"{image_name}: f{subtask_output}") + f"{image_name}: {subtask_output}") return build_status_dict, build_output_dict if __name__ == '__main__': + # initialise detection has_build, local_date, log_path, output_path = begin() if not has_build: # exit no. 2: no build today sys.exit(2) + + # find available log files main_logs, subtask_logs = find_log_files(local_date, log_path) solutions_paths = read_main_log(main_logs) + + # read build status and output information build_status, build_output = read_subtask_log( subtask_logs, solutions_paths) + + # TODO: use these information to construct report, currently just print for solution_producer in build_status.keys(): print('===================') solution, producer = solution_producer[0], solution_producer[1] print(f'Solution: {solution} Producer: {producer}') status_list = build_status.get(solution_producer) output_list = build_output.get(solution_producer) - for status, output in zip(build_status, build_output): + for status, output in zip(status_list, output_list): print(status) print(output) -- Gitee From b80d5d71c64a07e61e90393c4f509d3291c0118a Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 09:43:22 +0800 Subject: [PATCH 13/64] add: subtask class for easier output for debug and future develop --- generate/__init__.py | 0 generate/gen_report.py | 49 ++++++++++++++++++++++++------------------ generate/subtask.py | 19 ++++++++++++++++ 3 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 generate/__init__.py create mode 100644 generate/subtask.py diff --git a/generate/__init__.py b/generate/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/generate/gen_report.py b/generate/gen_report.py index ca7da61..bd7a517 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -7,6 +7,8 @@ import sys import time from typing import Dict, List, Tuple +from generate import subtask + def begin() -> Tuple[bool, str, str, str]: # generate local date in format 'YYYYMMDD', e.g. '20201202' @@ -66,13 +68,12 @@ def read_main_log(main_logs: List[str]) -> Dict[tuple, str]: def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ - tuple, str]) -> Tuple[Dict[tuple, List[str]], Dict[tuple, List[str]]]: + tuple, str], local_date: str) -> List[subtask.Subtask]: build_status_dict = {} build_output_dict = {} + subtasks = [] # check subtask logs by solution and producer for solution_producer, path in solution_path_dict.items(): - build_status_dict[solution_producer] = [] - build_output_dict[solution_producer] = [] for subtask_log in subtask_logs: if subtask_log.startswith(path): # this subtask log belongs to this solution/producer @@ -85,6 +86,7 @@ def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ break subtask_result = 'UNKNOWN' subtask_output = 'unknown' + subtask_error = '' for line in lines[::-1]: if "Build subtask output dir:" in line: subtask_output = line.split()[-1] @@ -92,13 +94,21 @@ def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ subtask_result = line.split()[-1] if subtask_result == 'FAIL': subtask_output = "Not Available" - break - # record status and output dir, the lists have same length - build_status_dict[solution_producer].append( - f"{image_name}: {subtask_result}") - build_output_dict[solution_producer].append( - f"{image_name}: {subtask_output}") - return build_status_dict, build_output_dict + length = len(lines) + # take last 12 lines for error reference + subtask_error = '\n'.join(lines[max(0, length - 12):]) + subtasks.append( + subtask.Subtask( + local_date, + subtask_log, + image_name, + solution_producer, + subtask_result, + subtask_output, + subtask_error + ) + ) + return subtasks if __name__ == '__main__': @@ -113,16 +123,13 @@ if __name__ == '__main__': solutions_paths = read_main_log(main_logs) # read build status and output information - build_status, build_output = read_subtask_log( - subtask_logs, solutions_paths) + subtasks = read_subtask_log(subtask_logs, solutions_paths, local_date) # TODO: use these information to construct report, currently just print - for solution_producer in build_status.keys(): - print('===================') - solution, producer = solution_producer[0], solution_producer[1] - print(f'Solution: {solution} Producer: {producer}') - status_list = build_status.get(solution_producer) - output_list = build_output.get(solution_producer) - for status, output in zip(status_list, output_list): - print(status) - print(output) + solution = '' + producer = '' + for task in subtasks: + if task.solution != solution or task.producer != producer: + solution, producer = task.solution, task.producer + print('===================') + print(str(task)) diff --git a/generate/subtask.py b/generate/subtask.py new file mode 100644 index 0000000..9b52fdd --- /dev/null +++ b/generate/subtask.py @@ -0,0 +1,19 @@ +class Subtask: + def __init__(self, date: str, log: str, image: str, + solution: tuple, status: str, output: str, error: str): + self.date = date + self.log = log + self.image = image + self.solution = solution[0] + self.producer = solution[1] + self.status = status + self.output = output + self.error = error + + def __str__(self): + if self.status == 'PASS': + result = f'{self.image} |\033[1;32m {self.status} \033[0m' + else: + result = f'{self.image} |\033[1;31;47m {self.status} \033[0m' + result += f'\n{self.error}' + return result -- Gitee From 2308eceb64177fa237dcaf6babcd8851ff71be34 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 10:10:22 +0800 Subject: [PATCH 14/64] Update: output of analyse result --- generate/gen_report.py | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index bd7a517..eb39d6a 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -7,7 +7,7 @@ import sys import time from typing import Dict, List, Tuple -from generate import subtask +from subtask import Subtask def begin() -> Tuple[bool, str, str, str]: @@ -68,7 +68,7 @@ def read_main_log(main_logs: List[str]) -> Dict[tuple, str]: def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ - tuple, str], local_date: str) -> List[subtask.Subtask]: + tuple, str], local_date: str) -> List[Subtask]: build_status_dict = {} build_output_dict = {} subtasks = [] @@ -96,9 +96,9 @@ def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ subtask_output = "Not Available" length = len(lines) # take last 12 lines for error reference - subtask_error = '\n'.join(lines[max(0, length - 12):]) + subtask_error = ''.join(lines[max(0, length - 12):]) subtasks.append( - subtask.Subtask( + Subtask( local_date, subtask_log, image_name, @@ -126,10 +126,27 @@ if __name__ == '__main__': subtasks = read_subtask_log(subtask_logs, solutions_paths, local_date) # TODO: use these information to construct report, currently just print - solution = '' - producer = '' + output = {} for task in subtasks: - if task.solution != solution or task.producer != producer: - solution, producer = task.solution, task.producer + if task.solution not in output: + output[task.solution] = {} + if task.producer not in output.get(task.solution): + output[task.solution][task.producer] = [] + output[task.solution][task.producer].append(task) + + for solution, producers in output.items: + for producer, tasks in producers.items: + passed, failed = 0, 0 + infos = [] + for task in tasks: + if task.status == 'PASS': + passed += 1 + else: + failed += 1 + infos.append(str(task)) print('===================') - print(str(task)) + pass_rate = "{:.2f}".format(passed / (passed + failed) * 100) + print(f'{solution} | {producer} | {passed} PASS, {failed} FAIL, {pass_rate}% PASS') + print('Details are shown below:') + for info in infos: + print(info) -- Gitee From 06cc670c33a4ce362b7f8e6795eec30891b12620 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 14:46:56 +0800 Subject: [PATCH 15/64] Add: write report and output info to files --- generate/gen_report.py | 108 ++++++++++++++++++++++++++++++----------- generate/subtask.py | 10 ++-- 2 files changed, 86 insertions(+), 32 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index eb39d6a..15c9cac 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -69,8 +69,6 @@ def read_main_log(main_logs: List[str]) -> Dict[tuple, str]: def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ tuple, str], local_date: str) -> List[Subtask]: - build_status_dict = {} - build_output_dict = {} subtasks = [] # check subtask logs by solution and producer for solution_producer, path in solution_path_dict.items(): @@ -79,6 +77,7 @@ def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ # this subtask log belongs to this solution/producer with open(subtask_log, 'r') as fp: lines = fp.readlines() + # read lines for information image_name = 'unknown' for line in lines: if "Build subtask image name:" in line: @@ -93,10 +92,11 @@ def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ if "Build subtask result:" in line: subtask_result = line.split()[-1] if subtask_result == 'FAIL': - subtask_output = "Not Available" + # if fail, extract last 10 lines of log for debug + subtask_output = "Not Available due to FAIL" length = len(lines) # take last 12 lines for error reference - subtask_error = ''.join(lines[max(0, length - 12):]) + subtask_error = ''.join(lines[max(0, length - 12):-2]) subtasks.append( Subtask( local_date, @@ -111,6 +111,76 @@ def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ return subtasks +def organise_tasks(subtasks): + # Solution: Producer: Tasks + solution_tasks = {} + for task in subtasks: + if task.solution not in solution_tasks: + solution_tasks[task.solution] = {} + if task.producer not in solution_tasks.get(task.solution): + solution_tasks[task.solution][task.producer] = [] + solution_tasks[task.solution][task.producer].append(task) + return solution_tasks + + +def generate_report(subtasks: List[Subtask]) -> Dict[str, str]: + solution_tasks = organise_tasks(subtasks) + solution_reports = {} + + for solution, producers in solution_tasks.items(): + report_infos = [] + for producer, tasks in producers.items(): + passed, failed = 0, 0 + infos = [] + for task in tasks: + if task.status == 'PASS': + passed += 1 + else: + failed += 1 + infos.append(str(task)) + # report output + report_infos.append('=' * 36 + '\n') + report_infos.append(f'Producer: {producer}\n') + pass_rate = "{:.2f}".format(passed / (passed + failed) * 100) + report_infos.append( + f'{producer} | {passed} PASS, {failed} FAIL, {pass_rate}% PASS\n') + report_infos.append('Details are shown below:\n') + report_infos.extend(infos) + solution_reports[solution] = ''.join(report_infos) + + return solution_reports + + +def generate_output(subtasks: List[Subtask]) -> Dict[str, str]: + solution_tasks = organise_tasks(subtasks) + solution_outputs = {} + + for solution, producers in solution_tasks.items(): + output_infos = [] + for producer, tasks in producers.items(): + infos = [] + for task in tasks: + infos.append(task.output_info()) + output_infos.append('=' * 36 + '\n') + output_infos.append(f'Producer: {producer}\n') + output_infos.extend(infos) + solution_outputs[solution] = ''.join(output_infos) + + return solution_outputs + + +def write_reports(local_date: str, reports: Dict[str, str], report_path: str) -> None: + for solution, report in reports.items(): + with open(f'{report_path}/{solution}_pipeline_report.txt', 'w') as fp: + fp.write(f'{solution} {local_date} pipeline build report\n\n' + report) + + +def write_outputs(local_date: str, outputs: Dict[str, str], output_path: str) -> None: + for solution, output in outputs.items(): + with open(f'{output_path}/{solution}_output_download.txt', 'w') as fp: + fp.write(f'{solution} {local_date} build output\n\n' + output) + + if __name__ == '__main__': # initialise detection has_build, local_date, log_path, output_path = begin() @@ -125,28 +195,8 @@ if __name__ == '__main__': # read build status and output information subtasks = read_subtask_log(subtask_logs, solutions_paths, local_date) - # TODO: use these information to construct report, currently just print - output = {} - for task in subtasks: - if task.solution not in output: - output[task.solution] = {} - if task.producer not in output.get(task.solution): - output[task.solution][task.producer] = [] - output[task.solution][task.producer].append(task) - - for solution, producers in output.items: - for producer, tasks in producers.items: - passed, failed = 0, 0 - infos = [] - for task in tasks: - if task.status == 'PASS': - passed += 1 - else: - failed += 1 - infos.append(str(task)) - print('===================') - pass_rate = "{:.2f}".format(passed / (passed + failed) * 100) - print(f'{solution} | {producer} | {passed} PASS, {failed} FAIL, {pass_rate}% PASS') - print('Details are shown below:') - for info in infos: - print(info) + reports = generate_report(subtasks) + outputs = generate_output(subtasks) + + write_reports(local_date, reports, f'/result/autotest/reports/{local_date}') + write_outputs(local_date, outputs, '/result/autotest/dc-build-latest') diff --git a/generate/subtask.py b/generate/subtask.py index 9b52fdd..c894062 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -12,8 +12,12 @@ class Subtask: def __str__(self): if self.status == 'PASS': - result = f'{self.image} |\033[1;32m {self.status} \033[0m' + result = f'{self.image} | {self.status}\n' else: - result = f'{self.image} |\033[1;31;47m {self.status} \033[0m' - result += f'\n{self.error}' + result = f'{self.image} | {self.status}\n' + result += f'{self.error}' + result += f'Full log available at: {self.log}\n' return result + + def output_info(self): + return f'{self.image}: {self.output}\n' -- Gitee From 75afb4d3da83eeaec94d66b4cea92ad0b399035a Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 14:51:13 +0800 Subject: [PATCH 16/64] Fix: create parent directory if not exist --- generate/gen_report.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 15c9cac..71ef4c5 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -170,12 +170,17 @@ def generate_output(subtasks: List[Subtask]) -> Dict[str, str]: def write_reports(local_date: str, reports: Dict[str, str], report_path: str) -> None: + if not os.path.exists(report_path): + os.mkdirs(report_path) for solution, report in reports.items(): - with open(f'{report_path}/{solution}_pipeline_report.txt', 'w') as fp: + file_path = f'{report_path}/{solution}_pipeline_report.txt' + with open(file_path, 'w') as fp: fp.write(f'{solution} {local_date} pipeline build report\n\n' + report) def write_outputs(local_date: str, outputs: Dict[str, str], output_path: str) -> None: + if not os.path.exists(output_path): + os.makedirs(output_path) for solution, output in outputs.items(): with open(f'{output_path}/{solution}_output_download.txt', 'w') as fp: fp.write(f'{solution} {local_date} build output\n\n' + output) -- Gitee From a79ebdb6c884618b63c1947ed3f11256cd8da2e0 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 15:17:41 +0800 Subject: [PATCH 17/64] Fix: makedirs bug, add read config for url --- generate/gen_report.py | 24 +++++++++++++++--------- generate/subtask.py | 16 ++++++++++++---- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 71ef4c5..09c6d75 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -139,11 +139,10 @@ def generate_report(subtasks: List[Subtask]) -> Dict[str, str]: failed += 1 infos.append(str(task)) # report output - report_infos.append('=' * 36 + '\n') - report_infos.append(f'Producer: {producer}\n') + report_infos.append(f'{"=" * 36} \nProducer: {producer}\n') pass_rate = "{:.2f}".format(passed / (passed + failed) * 100) - report_infos.append( - f'{producer} | {passed} PASS, {failed} FAIL, {pass_rate}% PASS\n') + report_infos.append(f'{producer} | {passed} PASS, ' + f'{failed} FAIL, {pass_rate}% PASS\n') report_infos.append('Details are shown below:\n') report_infos.extend(infos) solution_reports[solution] = ''.join(report_infos) @@ -169,16 +168,20 @@ def generate_output(subtasks: List[Subtask]) -> Dict[str, str]: return solution_outputs -def write_reports(local_date: str, reports: Dict[str, str], report_path: str) -> None: +def write_reports( + local_date: str, reports: Dict[str, str], report_path: str) -> None: if not os.path.exists(report_path): - os.mkdirs(report_path) + os.makedirs(report_path) for solution, report in reports.items(): file_path = f'{report_path}/{solution}_pipeline_report.txt' with open(file_path, 'w') as fp: - fp.write(f'{solution} {local_date} pipeline build report\n\n' + report) + fp.write( + f'{solution} {local_date} pipeline build report\n\n' + + report) -def write_outputs(local_date: str, outputs: Dict[str, str], output_path: str) -> None: +def write_outputs( + local_date: str, outputs: Dict[str, str], output_path: str) -> None: if not os.path.exists(output_path): os.makedirs(output_path) for solution, output in outputs.items(): @@ -203,5 +206,8 @@ if __name__ == '__main__': reports = generate_report(subtasks) outputs = generate_output(subtasks) - write_reports(local_date, reports, f'/result/autotest/reports/{local_date}') + write_reports( + local_date, + reports, + f'/result/autotest/reports/{local_date}') write_outputs(local_date, outputs, '/result/autotest/dc-build-latest') diff --git a/generate/subtask.py b/generate/subtask.py index c894062..015d059 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -1,22 +1,30 @@ +import yaml + + class Subtask: + server = '' + with open('config.yaml', 'r') as fp: + server = yaml.load(fp).get('address') + def __init__(self, date: str, log: str, image: str, solution: tuple, status: str, output: str, error: str): self.date = date - self.log = log + self.log = self.server + '/'.join(log.split('/')[2:]) self.image = image self.solution = solution[0] self.producer = solution[1] self.status = status self.output = output + if not self.output.startswith("Not Available"): + self.output = f'{self.server}/{self.date}/output/{self.output}' self.error = error def __str__(self): if self.status == 'PASS': result = f'{self.image} | {self.status}\n' else: - result = f'{self.image} | {self.status}\n' - result += f'{self.error}' - result += f'Full log available at: {self.log}\n' + result = (f'{self.image} | {self.status}\n{self.error}' + f'Full log available at: {self.log}\n') return result def output_info(self): -- Gitee From f6c15e5c378a25b31f633ac343b2d37acfcdbdfb Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 15:18:53 +0800 Subject: [PATCH 18/64] ignore yaml configuration file --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 440357d..3942eab 100644 --- a/.gitignore +++ b/.gitignore @@ -42,3 +42,4 @@ env.bak/ venv.bak/ .idea/ +.yaml -- Gitee From a96402a718c164b7d64925b8c9f20447e35f49f6 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 15:22:24 +0800 Subject: [PATCH 19/64] Fix: output url error --- generate/subtask.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/generate/subtask.py b/generate/subtask.py index 015d059..75e60fc 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -16,7 +16,8 @@ class Subtask: self.status = status self.output = output if not self.output.startswith("Not Available"): - self.output = f'{self.server}/{self.date}/output/{self.output}' + self.output = (f'{self.server}/actuators/{self.date}' + f'/output/{self.output}') self.error = error def __str__(self): -- Gitee From 011eac5f596c453a7f2eb6f751acd28aebb82033 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 15:27:10 +0800 Subject: [PATCH 20/64] Fix: change to json config --- .gitignore | 2 +- generate/subtask.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 3942eab..71ef654 100644 --- a/.gitignore +++ b/.gitignore @@ -42,4 +42,4 @@ env.bak/ venv.bak/ .idea/ -.yaml +.json diff --git a/generate/subtask.py b/generate/subtask.py index 75e60fc..520344c 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -1,10 +1,10 @@ -import yaml +import json class Subtask: server = '' - with open('config.yaml', 'r') as fp: - server = yaml.load(fp).get('address') + with open('config.json', 'r') as fp: + server = json.loads(fp.read()).get('address') def __init__(self, date: str, log: str, image: str, solution: tuple, status: str, output: str, error: str): -- Gitee From 5eb96a3e2abb8daae47064d27487de52908ad2fc Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 15:34:31 +0800 Subject: [PATCH 21/64] Fix: use absolute path for config read --- generate/subtask.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/generate/subtask.py b/generate/subtask.py index 520344c..b17bb9e 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -1,9 +1,11 @@ import json +import os class Subtask: server = '' - with open('config.json', 'r') as fp: + file_dir = os.path.dirname((os.path.abspath(__name__))) + with open(f'{file_dir}\config.json', 'r') as fp: server = json.loads(fp.read()).get('address') def __init__(self, date: str, log: str, image: str, -- Gitee From e841555da268f0c150e9b9e8ade1eb1f481b09b6 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 15:47:51 +0800 Subject: [PATCH 22/64] Fix: file path and url --- generate/subtask.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/generate/subtask.py b/generate/subtask.py index b17bb9e..d4c8f98 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -1,24 +1,24 @@ import json -import os +import sys class Subtask: server = '' - file_dir = os.path.dirname((os.path.abspath(__name__))) - with open(f'{file_dir}\config.json', 'r') as fp: + file_dir = sys.path[0] + with open(f'{file_dir}/config.json', 'r') as fp: server = json.loads(fp.read()).get('address') def __init__(self, date: str, log: str, image: str, solution: tuple, status: str, output: str, error: str): self.date = date - self.log = self.server + '/'.join(log.split('/')[2:]) + self.log = self.server + '/'.join(log.split('/')[3:]) self.image = image self.solution = solution[0] self.producer = solution[1] self.status = status self.output = output if not self.output.startswith("Not Available"): - self.output = (f'{self.server}/actuators/{self.date}' + self.output = (f'{self.server}actuators/{self.date}' f'/output/{self.output}') self.error = error -- Gitee From 4bb8742cd94c9abb1e5d148701a0d290e7fec2f4 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 16:03:24 +0800 Subject: [PATCH 23/64] Fix: url path error. Add: instruction --- README.md | 38 ++++++++++++-------------------------- dispatch/.keep | 0 generate/subtask.py | 2 +- 3 files changed, 13 insertions(+), 27 deletions(-) create mode 100644 dispatch/.keep diff --git a/README.md b/README.md index c7968ac..53b98aa 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,23 @@ -# report +# 编译日志报告生成 #### 介绍 -归档日志自动生成报告，发送邮件通知 -#### 软件架构 -软件架构说明 +归档日志自动生成报告，发送邮件通知 #### 安装教程 -1. xxxx -2. xxxx -3. xxxx - -#### 使用说明 - -1. xxxx -2. xxxx -3. xxxx - -#### 参与贡献 +1. 克隆此仓库到本地 +2. 为服务器URL提示创建配置文件 generate/config.json，如下： +```json +{ + "address": "example.com/archive_path/" +} +``` -1. Fork 本仓库 -2. 新建 Feat_xxx 分支 -3. 提交代码 -4. 新建 Pull Request +#### 使用教程 -#### 特技 +1. 执行 generate/gen_report.py 以生成报告等 +2. 执行 xxxx 以发送邮件通知 -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md -2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com) -3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目 -4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目，是综合评定出的优秀开源项目 -5. Gitee 官方提供的使用手册 [https://gitee.com/help](https://gitee.com/help) -6. Gitee 封面人物是一档用来展示 Gitee 会员风采的栏目 [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/dispatch/.keep b/dispatch/.keep new file mode 100644 index 0000000..e69de29 diff --git a/generate/subtask.py b/generate/subtask.py index d4c8f98..0a50711 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -19,7 +19,7 @@ class Subtask: self.output = output if not self.output.startswith("Not Available"): self.output = (f'{self.server}actuators/{self.date}' - f'/output/{self.output}') + f'/output/{self.output}/') self.error = error def __str__(self): -- Gitee From 79a534af10c9d5cf37596808405be65948486c7a Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 19:56:31 +0800 Subject: [PATCH 24/64] Update: reorganise report build process --- README.md | 2 +- generate/gen_report.py | 166 +++++++++++++++++++++-------------------- generate/maintask.py | 57 ++++++++++++++ generate/subtask.py | 36 +++++---- 4 files changed, 168 insertions(+), 93 deletions(-) create mode 100644 generate/maintask.py diff --git a/README.md b/README.md index 53b98aa..beaf2f1 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ 归档日志自动生成报告，发送邮件通知 -#### 安装教程 +#### 1. 克隆此仓库到本地 2. 为服务器URL提示创建配置文件 generate/config.json，如下： diff --git a/generate/gen_report.py b/generate/gen_report.py index 09c6d75..759ad4c 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -7,9 +7,11 @@ import sys import time from typing import Dict, List, Tuple +from maintask import Maintask from subtask import Subtask + def begin() -> Tuple[bool, str, str, str]: # generate local date in format 'YYYYMMDD', e.g. '20201202' local_date = time.strftime('%Y%m%d', time.localtime(time.time())) @@ -40,128 +42,134 @@ def find_log_files( return main_logs, subtask_logs -def read_main_log(main_logs: List[str]) -> Dict[tuple, str]: - solution_path_dict = {} +def read_main_log(main_logs: List[str]) -> List[Maintask]: + maintasks = [] for log_file in main_logs: with open(log_file, 'r') as fp: lines = fp.readlines() - # find solution and producer - solution = "" - producer = "" + # find information + task = '' + build_os = '' + arch = '' + docker = '' + solution = '' + producer = '' + version = '' + start_time = '' + end_time = '' for line in lines: + if 'Actuator OS' in line: + build_os = line.split()[3] + if 'Actuator ARCH' in line: + arch = line.split()[3] + if 'Actuator Docker' in line: + docker = line.split()[3] if 'Solution:' in line: solution = line.split()[3] if 'Producer:' in line: producer = line.split()[3] + if 'Version:' in line: + version = line.split()[3] + if 'Build PATH' in line: + task = line.split()[3] + f'/{version}' + if 'Build task start time' in line: + start_time = line.split(':')[1].strip() + if 'Build task end time' in line: + end_time = line.split(':')[1].strip() + cur_task = Maintask(task, log_file, build_os, arch, docker, solution, producer, version, start_time, end_time) + maintasks.append(cur_task) - if not solution: - # cannot detect solution, call it unknown - solution = "Unknown" - if not producer: - # cannot detect producer, call it unknown - producer = "Unknown" - - # record path of the main log - subtask logs will be in here - solution_path_dict[(solution, producer)] = os.path.dirname(log_file) - return solution_path_dict + return maintasks -def read_subtask_log(subtask_logs: List[str], solution_path_dict: Dict[ - tuple, str], local_date: str) -> List[Subtask]: - subtasks = [] +def read_subtask_log(subtask_logs: List[str], main_tasks: List[Maintask], local_date: str) -> List[Maintask]: # check subtask logs by solution and producer - for solution_producer, path in solution_path_dict.items(): + for main_task in main_tasks: for subtask_log in subtask_logs: - if subtask_log.startswith(path): + if subtask_log.startswith(main_task.log_dir): # this subtask log belongs to this solution/producer with open(subtask_log, 'r') as fp: lines = fp.readlines() # read lines for information image_name = 'unknown' - for line in lines: - if "Build subtask image name:" in line: - image_name = line.split()[-1].split(':')[0] - break subtask_result = 'UNKNOWN' subtask_output = 'unknown' subtask_error = '' - for line in lines[::-1]: + subtask_start = '1970-01-01 00:00:00' + subtask_end = '1970-01-01 00:00:00' + for line in lines: + if "Build subtask image name:" in line: + image_name = line.split()[-1].split(':')[0] if "Build subtask output dir:" in line: subtask_output = line.split()[-1] if "Build subtask result:" in line: subtask_result = line.split()[-1] - if subtask_result == 'FAIL': - # if fail, extract last 10 lines of log for debug - subtask_output = "Not Available due to FAIL" - length = len(lines) - # take last 12 lines for error reference - subtask_error = ''.join(lines[max(0, length - 12):-2]) - subtasks.append( + if 'Build task start time' in line: + subtask_start = line.split(':')[1].strip() + if 'Build task end time' in line: + subtask_end = line.split(':')[1].strip() + + if subtask_result == 'FAIL': + # if fail, extract last 10 lines of log for debug + subtask_output = "Not Available due to FAIL" + length = len(lines) + # take last 12 lines for error reference + subtask_error = ''.join(lines[max(0, length - 12):-2]) + + main_task.add_subtask( Subtask( local_date, subtask_log, image_name, - solution_producer, + main_task.solution, + main_task.producer, subtask_result, subtask_output, - subtask_error + subtask_error, + subtask_start, + subtask_end ) ) - return subtasks + return main_tasks -def organise_tasks(subtasks): - # Solution: Producer: Tasks +def organise_tasks(main_tasks: List[Maintask]) -> Dict[str, List[Maintask]]: + # Solution: Tasks solution_tasks = {} - for task in subtasks: + for task in main_tasks: if task.solution not in solution_tasks: - solution_tasks[task.solution] = {} - if task.producer not in solution_tasks.get(task.solution): - solution_tasks[task.solution][task.producer] = [] - solution_tasks[task.solution][task.producer].append(task) + solution_tasks[task.solution] = [] + solution_tasks[task.solution].append(task) return solution_tasks -def generate_report(subtasks: List[Subtask]) -> Dict[str, str]: - solution_tasks = organise_tasks(subtasks) +def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: + solution_tasks = organise_tasks(main_tasks) solution_reports = {} - - for solution, producers in solution_tasks.items(): - report_infos = [] - for producer, tasks in producers.items(): - passed, failed = 0, 0 - infos = [] - for task in tasks: - if task.status == 'PASS': - passed += 1 - else: - failed += 1 - infos.append(str(task)) - # report output - report_infos.append(f'{"=" * 36} \nProducer: {producer}\n') - pass_rate = "{:.2f}".format(passed / (passed + failed) * 100) - report_infos.append(f'{producer} | {passed} PASS, ' - f'{failed} FAIL, {pass_rate}% PASS\n') - report_infos.append('Details are shown below:\n') - report_infos.extend(infos) - solution_reports[solution] = ''.join(report_infos) + for solution, sol_main_tasks in solution_tasks.items(): + report_infos_brief = [] + report_infos_detail = [] + for task in sol_main_tasks: + report_infos_brief.append(str(task) + task.subtasks_info_general()) + report_infos_detail.append(task.subtasks_info_detail()) + solution_reports[solution] = '\n'.join(report_infos_brief) + f'\n{"=" * 80}\nDetail Report\n' + report_infos_detail return solution_reports -def generate_output(subtasks: List[Subtask]) -> Dict[str, str]: - solution_tasks = organise_tasks(subtasks) +def generate_output(main_tasks: List[Maintask]) -> Dict[str, str]: + solution_tasks = organise_tasks(main_tasks) solution_outputs = {} - for solution, producers in solution_tasks.items(): + for solution, main_tasks in solution_tasks.items(): output_infos = [] - for producer, tasks in producers.items(): + for main_task in main_tasks: infos = [] - for task in tasks: + for task in main_task.subtasks: infos.append(task.output_info()) - output_infos.append('=' * 36 + '\n') - output_infos.append(f'Producer: {producer}\n') + output_infos.append('=' * 80 + '\n') + output_infos.append(f'Producer: {main_task.producer}\n') output_infos.extend(infos) solution_outputs[solution] = ''.join(output_infos) @@ -173,10 +181,10 @@ def write_reports( if not os.path.exists(report_path): os.makedirs(report_path) for solution, report in reports.items(): - file_path = f'{report_path}/{solution}_pipeline_report.txt' + file_path = f'{report_path}/{solution}_daily_report' with open(file_path, 'w') as fp: fp.write( - f'{solution} {local_date} pipeline build report\n\n' + + f'\t{solution} {local_date} Build Report\n\n' + report) @@ -185,8 +193,8 @@ def write_outputs( if not os.path.exists(output_path): os.makedirs(output_path) for solution, output in outputs.items(): - with open(f'{output_path}/{solution}_output_download.txt', 'w') as fp: - fp.write(f'{solution} {local_date} build output\n\n' + output) + with open(f'{output_path}/{solution}_daily_output', 'w') as fp: + fp.write(f'\t{solution} {local_date} build output\n\n' + output) if __name__ == '__main__': @@ -198,13 +206,13 @@ if __name__ == '__main__': # find available log files main_logs, subtask_logs = find_log_files(local_date, log_path) - solutions_paths = read_main_log(main_logs) + main_tasks = read_main_log(main_logs) # read build status and output information - subtasks = read_subtask_log(subtask_logs, solutions_paths, local_date) + main_tasks = read_subtask_log(subtask_logs, main_tasks, local_date) - reports = generate_report(subtasks) - outputs = generate_output(subtasks) + reports = generate_report(main_tasks) + outputs = generate_output(main_tasks) write_reports( local_date, diff --git a/generate/maintask.py b/generate/maintask.py new file mode 100644 index 0000000..920cb6d --- /dev/null +++ b/generate/maintask.py @@ -0,0 +1,57 @@ +import json +import sys +import time +from subtask import Subtask + + +class Maintask: + server = '' + file_dir = sys.path[0] + with open(f'{file_dir}/config.json', 'r') as fp: + server = json.loads(fp.read()).get('address') + + def __init__(self, task, log, os, arch, docker, solution, producer, version, start_time, end_time): + self.task = task + self.log = self.server + '/'.join(log.split('/')[3:]) + self.log_dir = os.path.dirname(self.log) + self.os = os + self.arch = arch + self.docker = docker + self.solution = solution + self.producer = producer + self.version = version + self.start_time = start_time + self.end_time = end_time + start = time.strptime(self.start_time, '%Y-%m-%d %H:%M:%S') + end = time.strptime(self.end_time, '%Y-%m-%d %H:%M:%S') + delta = end - start + self.duration = str(delta) + self.subtasks = [] + self.success = 0 + self.fail = 0 + self.result = 'PASS' + + def __str__(self): + passrate = "{:.1f}".format(self.success / (self.success + self.fail) * 100) + return f"""{'=' * 80} +Task Name: {self.task} +logfile: {self.log} +OS: {self.os} +Architecture: {self.arch} +docker: {self.docker} +Time: {self.start_time} ~ {self.end_time}, Cost {self.duration} +Result: {self.result}. Subtasks: {self.success} PASS {self.fail} FAIL; PASS RATE {passrate} +""" + + def subtasks_info_general(self): + return '\n'.join([subtask.general_status() for subtask in self.subtasks]) + + def subtasks_info_detail(self): + return '\n'.join([subtask.detail_status() for subtask in self.subtasks]) + + def add_subtask(self, sub_task: Subtask): + if sub_task.status == 'PASS': + self.success += 1 + else: + self.fail += 1 + self.subtasks.append(sub_task) diff --git a/generate/subtask.py b/generate/subtask.py index 0a50711..a739463 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -1,5 +1,6 @@ import json import sys +import time class Subtask: @@ -8,27 +9,36 @@ class Subtask: with open(f'{file_dir}/config.json', 'r') as fp: server = json.loads(fp.read()).get('address') - def __init__(self, date: str, log: str, image: str, - solution: tuple, status: str, output: str, error: str): + def __init__(self, date, log, image, solution, producer, status, output, error, start_time, end_time): self.date = date self.log = self.server + '/'.join(log.split('/')[3:]) self.image = image - self.solution = solution[0] - self.producer = solution[1] + self.solution = solution + self.producer = producer self.status = status self.output = output if not self.output.startswith("Not Available"): self.output = (f'{self.server}actuators/{self.date}' f'/output/{self.output}/') - self.error = error - - def __str__(self): - if self.status == 'PASS': - result = f'{self.image} | {self.status}\n' - else: - result = (f'{self.image} | {self.status}\n{self.error}' - f'Full log available at: {self.log}\n') - return result + self.error = '\t' + error.replace('\n', '\n\t') + self.start_time = start_time + self.end_time = end_time + start = time.strptime(self.start_time, '%Y-%m-%d %H:%M:%S') + end = time.strptime(self.end_time, '%Y-%m-%d %H:%M:%S') + delta = end - start + self.duration = str(delta) def output_info(self): return f'{self.image}: {self.output}\n' + + def general_status(self): + return f'\t{self.producer}\t{self.image}\t{self.status}' + + def detail_status(self): + return f"""{self.image}: {self.status} +Producer: {self.producer} +Log: {self.log} +Time: {self.start_time} ~ {self.end_time}, Cost {self.duration} +{'Help message: (last 10 lines of log)' if self.status == 'FAIL' else ''} +{self.error if self.status == 'FAIL' else ''} +""" -- Gitee From cb92c72210969dfdc05495fafbf2aab075b590ab Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 20:00:01 +0800 Subject: [PATCH 25/64] Fix: os name shadow error --- generate/maintask.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/generate/maintask.py b/generate/maintask.py index 920cb6d..7723156 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -1,4 +1,5 @@ import json +import os import sys import time from subtask import Subtask @@ -10,11 +11,11 @@ class Maintask: with open(f'{file_dir}/config.json', 'r') as fp: server = json.loads(fp.read()).get('address') - def __init__(self, task, log, os, arch, docker, solution, producer, version, start_time, end_time): + def __init__(self, task, log, curr_os, arch, docker, solution, producer, version, start_time, end_time): self.task = task self.log = self.server + '/'.join(log.split('/')[3:]) self.log_dir = os.path.dirname(self.log) - self.os = os + self.os = curr_os self.arch = arch self.docker = docker self.solution = solution -- Gitee From b7f09102149e0c25c5326a2ed08decf62fca54b5 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 20:05:43 +0800 Subject: [PATCH 26/64] Fix: time extract error --- generate/gen_report.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 759ad4c..0b12190 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -74,9 +74,9 @@ def read_main_log(main_logs: List[str]) -> List[Maintask]: if 'Build PATH' in line: task = line.split()[3] + f'/{version}' if 'Build task start time' in line: - start_time = line.split(':')[1].strip() + start_time = ' '.join(line.split()[-2:]).strip() if 'Build task end time' in line: - end_time = line.split(':')[1].strip() + end_time = ' '.join(line.split()[-2:]).strip() cur_task = Maintask(task, log_file, build_os, arch, docker, solution, producer, version, start_time, end_time) maintasks.append(cur_task) @@ -106,9 +106,9 @@ def read_subtask_log(subtask_logs: List[str], main_tasks: List[Maintask], local_ if "Build subtask result:" in line: subtask_result = line.split()[-1] if 'Build task start time' in line: - subtask_start = line.split(':')[1].strip() + subtask_start = ' '.join(line.split()[-2:]).strip() if 'Build task end time' in line: - subtask_end = line.split(':')[1].strip() + subtask_end = ' '.join(line.split()[-2:]).strip() if subtask_result == 'FAIL': # if fail, extract last 10 lines of log for debug -- Gitee From 9f713201708f236e688f18354ab9e3b6567ee2c2 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 20:10:35 +0800 Subject: [PATCH 27/64] Fix: time substraction error, use datetime --- generate/maintask.py | 7 ++++--- generate/subtask.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/generate/maintask.py b/generate/maintask.py index 7723156..3a6fcb6 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -1,7 +1,8 @@ +from datetime import datetime import json import os import sys -import time + from subtask import Subtask @@ -23,8 +24,8 @@ class Maintask: self.version = version self.start_time = start_time self.end_time = end_time - start = time.strptime(self.start_time, '%Y-%m-%d %H:%M:%S') - end = time.strptime(self.end_time, '%Y-%m-%d %H:%M:%S') + start = datetime.strptime(self.start_time, '%Y-%m-%d %H:%M:%S') + end = datetime.strptime(self.end_time, '%Y-%m-%d %H:%M:%S') delta = end - start self.duration = str(delta) self.subtasks = [] diff --git a/generate/subtask.py b/generate/subtask.py index a739463..b50cc11 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -1,6 +1,7 @@ +from datetime import datetime import json import sys -import time + class Subtask: @@ -23,8 +24,8 @@ class Subtask: self.error = '\t' + error.replace('\n', '\n\t') self.start_time = start_time self.end_time = end_time - start = time.strptime(self.start_time, '%Y-%m-%d %H:%M:%S') - end = time.strptime(self.end_time, '%Y-%m-%d %H:%M:%S') + start = datetime.strptime(self.start_time, '%Y-%m-%d %H:%M:%S') + end = datetime.strptime(self.end_time, '%Y-%m-%d %H:%M:%S') delta = end - start self.duration = str(delta) -- Gitee From 9d39d75895c3a49943ef3dc445a071f4baa4f0b5 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 20:19:18 +0800 Subject: [PATCH 28/64] Fix: main log dir, output --- generate/gen_report.py | 2 +- generate/maintask.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 0b12190..0ab7022 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -153,7 +153,7 @@ def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: for task in sol_main_tasks: report_infos_brief.append(str(task) + task.subtasks_info_general()) report_infos_detail.append(task.subtasks_info_detail()) - solution_reports[solution] = '\n'.join(report_infos_brief) + f'\n{"=" * 80}\nDetail Report\n' + report_infos_detail + solution_reports[solution] = '\n'.join(report_infos_brief) + f'\n{"=" * 80}\nDetail Report\n' + '\n'.join(report_infos_detail) return solution_reports diff --git a/generate/maintask.py b/generate/maintask.py index 3a6fcb6..e03d669 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -15,7 +15,7 @@ class Maintask: def __init__(self, task, log, curr_os, arch, docker, solution, producer, version, start_time, end_time): self.task = task self.log = self.server + '/'.join(log.split('/')[3:]) - self.log_dir = os.path.dirname(self.log) + self.log_dir = os.path.dirname(log) self.os = curr_os self.arch = arch self.docker = docker -- Gitee From f871f630f03fd72820eed61bd4096c93b2ed3b8b Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 20:21:48 +0800 Subject: [PATCH 29/64] Fix: info extract error --- generate/gen_report.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 0ab7022..1dc2d84 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -60,19 +60,19 @@ def read_main_log(main_logs: List[str]) -> List[Maintask]: end_time = '' for line in lines: if 'Actuator OS' in line: - build_os = line.split()[3] + build_os = line.split()[-1] if 'Actuator ARCH' in line: - arch = line.split()[3] + arch = line.split()[-1] if 'Actuator Docker' in line: - docker = line.split()[3] + docker = line.split()[-1] if 'Solution:' in line: - solution = line.split()[3] + solution = line.split()[-1] if 'Producer:' in line: - producer = line.split()[3] + producer = line.split()[-1] if 'Version:' in line: - version = line.split()[3] + version = line.split()[-1] if 'Build PATH' in line: - task = line.split()[3] + f'/{version}' + task = line.split()[-1] + f'/{version}' if 'Build task start time' in line: start_time = ' '.join(line.split()[-2:]).strip() if 'Build task end time' in line: -- Gitee From cd722b4f4757c49f83c33f273f58211656558b5a Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 20:23:13 +0800 Subject: [PATCH 30/64] Fix: overall result error --- generate/maintask.py | 1 + 1 file changed, 1 insertion(+) diff --git a/generate/maintask.py b/generate/maintask.py index e03d669..93dacad 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -56,4 +56,5 @@ Result: {self.result}. Subtasks: {self.success} PASS {self.fail} FAIL; PASS RATE self.success += 1 else: self.fail += 1 + self.result = 'FAIL' self.subtasks.append(sub_task) -- Gitee From 18924337df09c1811cfd98dc4ab45d8b906b29eb Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 20:25:30 +0800 Subject: [PATCH 31/64] Fix: subtask time extract error --- generate/gen_report.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 1dc2d84..9174142 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -105,9 +105,9 @@ def read_subtask_log(subtask_logs: List[str], main_tasks: List[Maintask], local_ subtask_output = line.split()[-1] if "Build subtask result:" in line: subtask_result = line.split()[-1] - if 'Build task start time' in line: + if 'Build subtask start time' in line: subtask_start = ' '.join(line.split()[-2:]).strip() - if 'Build task end time' in line: + if 'Build subtask end time' in line: subtask_end = ' '.join(line.split()[-2:]).strip() if subtask_result == 'FAIL': -- Gitee From fc2f27bea46765c41b7da26a60596596cf1a21e5 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Thu, 3 Dec 2020 20:28:25 +0800 Subject: [PATCH 32/64] Fix: os info extract error --- generate/gen_report.py | 2 +- generate/maintask.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 9174142..33d1bd9 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -60,7 +60,7 @@ def read_main_log(main_logs: List[str]) -> List[Maintask]: end_time = '' for line in lines: if 'Actuator OS' in line: - build_os = line.split()[-1] + build_os = ' '.join(line.split()[4:]).strip() if 'Actuator ARCH' in line: arch = line.split()[-1] if 'Actuator Docker' in line: diff --git a/generate/maintask.py b/generate/maintask.py index 93dacad..b043cf3 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -42,7 +42,7 @@ OS: {self.os} Architecture: {self.arch} docker: {self.docker} Time: {self.start_time} ~ {self.end_time}, Cost {self.duration} -Result: {self.result}. Subtasks: {self.success} PASS {self.fail} FAIL; PASS RATE {passrate} +Result: {self.result}. Subtasks: {self.success} PASS {self.fail} FAIL; PASS RATE {passrate}% """ def subtasks_info_general(self): -- Gitee From bb411430cf82bf71940dcc6a0ab77a8b12f7a93e Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 09:28:22 +0800 Subject: [PATCH 33/64] Enhancement: More readable report generate --- generate/gen_report.py | 7 ++++--- generate/maintask.py | 15 +++++++++++---- generate/subtask.py | 18 +++++++++++------- 3 files changed, 26 insertions(+), 14 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 33d1bd9..cc727d7 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -11,7 +11,6 @@ from maintask import Maintask from subtask import Subtask - def begin() -> Tuple[bool, str, str, str]: # generate local date in format 'YYYYMMDD', e.g. '20201202' local_date = time.strftime('%Y%m%d', time.localtime(time.time())) @@ -72,7 +71,7 @@ def read_main_log(main_logs: List[str]) -> List[Maintask]: if 'Version:' in line: version = line.split()[-1] if 'Build PATH' in line: - task = line.split()[-1] + f'/{version}' + task = line.split()[-1] if 'Build task start time' in line: start_time = ' '.join(line.split()[-2:]).strip() if 'Build task end time' in line: @@ -153,7 +152,9 @@ def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: for task in sol_main_tasks: report_infos_brief.append(str(task) + task.subtasks_info_general()) report_infos_detail.append(task.subtasks_info_detail()) - solution_reports[solution] = '\n'.join(report_infos_brief) + f'\n{"=" * 80}\nDetail Report\n' + '\n'.join(report_infos_detail) + solution_reports[solution] = '\n'.join(report_infos_brief) \ + + f'\n\n{"=" * 120}\n\t[Detail Report]\n' \ + + '\n'.join(report_infos_detail) return solution_reports diff --git a/generate/maintask.py b/generate/maintask.py index b043cf3..185dfb8 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -10,7 +10,11 @@ class Maintask: server = '' file_dir = sys.path[0] with open(f'{file_dir}/config.json', 'r') as fp: - server = json.loads(fp.read()).get('address') + try: + server = json.loads(fp.read()).get('address') + except KeyError: + # no address provided, use local + server = "/" def __init__(self, task, log, curr_os, arch, docker, solution, producer, version, start_time, end_time): self.task = task @@ -37,6 +41,7 @@ class Maintask: passrate = "{:.1f}".format(self.success / (self.success + self.fail) * 100) return f"""{'=' * 80} Task Name: {self.task} +Version: {self.version} logfile: {self.log} OS: {self.os} Architecture: {self.arch} @@ -46,15 +51,17 @@ Result: {self.result}. Subtasks: {self.success} PASS {self.fail} FAIL; PASS RATE """ def subtasks_info_general(self): - return '\n'.join([subtask.general_status() for subtask in self.subtasks]) + sub_task_general = sorted([subtask.general_status() for subtask in self.subtasks]) + sub_task_general = ['\tProducer\tImage Name\t\t\tBuild Status'] + sub_task_general + return '\n'.join(sub_task_general) def subtasks_info_detail(self): - return '\n'.join([subtask.detail_status() for subtask in self.subtasks]) + return '\n'.join(sorted([subtask.detail_status() for subtask in self.subtasks])) def add_subtask(self, sub_task: Subtask): if sub_task.status == 'PASS': self.success += 1 else: self.fail += 1 - self.result = 'FAIL' + self.result = '**FAIL**' self.subtasks.append(sub_task) diff --git a/generate/subtask.py b/generate/subtask.py index b50cc11..269c3ce 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -3,14 +3,18 @@ import json import sys - class Subtask: server = '' file_dir = sys.path[0] with open(f'{file_dir}/config.json', 'r') as fp: - server = json.loads(fp.read()).get('address') + try: + server = json.loads(fp.read()).get('address') + except KeyError: + # no address provided, use local + server = "/" - def __init__(self, date, log, image, solution, producer, status, output, error, start_time, end_time): + def __init__(self, date, log, image, solution, producer, + status, output, error, start_time, end_time): self.date = date self.log = self.server + '/'.join(log.split('/')[3:]) self.image = image @@ -33,13 +37,13 @@ class Subtask: return f'{self.image}: {self.output}\n' def general_status(self): - return f'\t{self.producer}\t{self.image}\t{self.status}' + return f'\t{self.producer}\t{self.image}\t{"**FAIL**" if self.status != "PASS" else "PASS"}' def detail_status(self): - return f"""{self.image}: {self.status} + newline = '\n' + return f"""{self.image}: {'**FAIL**' if self.status != 'PASS' else 'PASS'} Producer: {self.producer} Log: {self.log} Time: {self.start_time} ~ {self.end_time}, Cost {self.duration} -{'Help message: (last 10 lines of log)' if self.status == 'FAIL' else ''} -{self.error if self.status == 'FAIL' else ''} +{newline.join(('Help message: (last 10 lines of log)', self.error)) if self.status != 'PASS' else ''} """ -- Gitee From 5094c69f0d06160495008af87a146eaa0cfe71fb Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 09:42:49 +0800 Subject: [PATCH 34/64] Add: specified date call on generate --- generate/gen_report.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index cc727d7..d688f2a 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -11,9 +11,10 @@ from maintask import Maintask from subtask import Subtask -def begin() -> Tuple[bool, str, str, str]: +def begin(local_date='') -> Tuple[bool, str, str, str]: # generate local date in format 'YYYYMMDD', e.g. '20201202' - local_date = time.strftime('%Y%m%d', time.localtime(time.time())) + if not local_date: + local_date = time.strftime('%Y%m%d', time.localtime(time.time())) # construct log and output path for today log_path = f'/result/autotest/actuators/{local_date}/log' output_path = f'/result/autotest/actuators/{local_date}/output' @@ -199,8 +200,16 @@ def write_outputs( if __name__ == '__main__': + args = sys.argv[1:] + date = '' + if args: + # specify generate date + date = args[0] # initialise detection - has_build, local_date, log_path, output_path = begin() + if date: + has_build, local_date, log_path, output_path = begin(local_date=date) + else: + has_build, local_date, log_path, output_path = begin() if not has_build: # exit no. 2: no build today sys.exit(2) @@ -219,4 +228,6 @@ if __name__ == '__main__': local_date, reports, f'/result/autotest/reports/{local_date}') - write_outputs(local_date, outputs, '/result/autotest/dc-build-latest') + if not args: + # do not overwrite output when called specified date + write_outputs(local_date, outputs, '/result/autotest/dc-build-latest') -- Gitee From f499a1f03aab96f2e00e67c6ac848ee66708f0af Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 09:50:40 +0800 Subject: [PATCH 35/64] Enhancement: enhance output format --- generate/gen_report.py | 2 +- generate/maintask.py | 2 +- generate/subtask.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index d688f2a..864c77e 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -154,7 +154,7 @@ def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: report_infos_brief.append(str(task) + task.subtasks_info_general()) report_infos_detail.append(task.subtasks_info_detail()) solution_reports[solution] = '\n'.join(report_infos_brief) \ - + f'\n\n{"=" * 120}\n\t[Detail Report]\n' \ + + f'\n\n{"=" * 120}\n\t\t[Detail Report]\n\n' \ + '\n'.join(report_infos_detail) return solution_reports diff --git a/generate/maintask.py b/generate/maintask.py index 185dfb8..7d2fd5c 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -52,7 +52,7 @@ Result: {self.result}. Subtasks: {self.success} PASS {self.fail} FAIL; PASS RATE def subtasks_info_general(self): sub_task_general = sorted([subtask.general_status() for subtask in self.subtasks]) - sub_task_general = ['\tProducer\tImage Name\t\t\tBuild Status'] + sub_task_general + sub_task_general = ['\tProducer\tImage Name\t\t\t\tBuild Status'] + sub_task_general return '\n'.join(sub_task_general) def subtasks_info_detail(self): diff --git a/generate/subtask.py b/generate/subtask.py index 269c3ce..18ba33f 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -37,7 +37,7 @@ class Subtask: return f'{self.image}: {self.output}\n' def general_status(self): - return f'\t{self.producer}\t{self.image}\t{"**FAIL**" if self.status != "PASS" else "PASS"}' + return f'\t{self.producer:16s}{self.image:42s}{"**FAIL**" if self.status != "PASS" else "PASS"}' def detail_status(self): newline = '\n' -- Gitee From 61b3942f038a4b4c1811829c624d98183c5d9b8c Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 09:57:41 +0800 Subject: [PATCH 36/64] Enhancement: report formatting, put proper empty lines there --- generate/gen_report.py | 4 ++-- generate/maintask.py | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 864c77e..aba8cde 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -154,7 +154,7 @@ def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: report_infos_brief.append(str(task) + task.subtasks_info_general()) report_infos_detail.append(task.subtasks_info_detail()) solution_reports[solution] = '\n'.join(report_infos_brief) \ - + f'\n\n{"=" * 120}\n\t\t[Detail Report]\n\n' \ + + f'\n\n{"=" * 120}\n\t[Subtask Detail Report]\n\n' \ + '\n'.join(report_infos_detail) return solution_reports @@ -186,7 +186,7 @@ def write_reports( file_path = f'{report_path}/{solution}_daily_report' with open(file_path, 'w') as fp: fp.write( - f'\t{solution} {local_date} Build Report\n\n' + + f'\t\t{solution} {local_date} Build Report\n' + report) diff --git a/generate/maintask.py b/generate/maintask.py index 7d2fd5c..3a45b99 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -39,15 +39,16 @@ class Maintask: def __str__(self): passrate = "{:.1f}".format(self.success / (self.success + self.fail) * 100) - return f"""{'=' * 80} + return f"""\n{'=' * 80} Task Name: {self.task} Version: {self.version} -logfile: {self.log} +Logfile: {self.log} OS: {self.os} Architecture: {self.arch} -docker: {self.docker} -Time: {self.start_time} ~ {self.end_time}, Cost {self.duration} +Docker version: {self.docker} +Build time: {self.start_time} ~ {self.end_time}, Cost {self.duration} Result: {self.result}. Subtasks: {self.success} PASS {self.fail} FAIL; PASS RATE {passrate}% +Subtask(s) build Status: """ def subtasks_info_general(self): -- Gitee From 138baa1ed38dab7769cbdff09d17d59d9e98bd87 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 11:08:08 +0800 Subject: [PATCH 37/64] Add: smtp dispatch of report --- dispatch/{.keep => __init__.py} | 0 dispatch/send_report.py | 45 +++++++++++++++++++++++++ dispatch/smtp.py | 60 +++++++++++++++++++++++++++++++++ generate/gen_report.py | 9 +---- 4 files changed, 106 insertions(+), 8 deletions(-) rename dispatch/{.keep => __init__.py} (100%) create mode 100644 dispatch/send_report.py create mode 100644 dispatch/smtp.py diff --git a/dispatch/.keep b/dispatch/__init__.py similarity index 100% rename from dispatch/.keep rename to dispatch/__init__.py diff --git a/dispatch/send_report.py b/dispatch/send_report.py new file mode 100644 index 0000000..12a3ca7 --- /dev/null +++ b/dispatch/send_report.py @@ -0,0 +1,45 @@ +#!/usr/bin/python3 +import json +import sys +import time +from typing import Dict + +from smtp import SMTP + + +def read_solution_email_receiver() -> Dict[str, str]: + file_dir = sys.path[0] + solution_receivers = {} + with open(f'{file_dir}/config.json', 'r') as fp: + config = json.loads(fp.read()) + try: + for key, value in config.items(): + solution_receivers[key] = value + except AttributeError as e: + # config file not valid + return {} + return solution_receivers + + +def dispatch_email(date, solution_receivers, smtp_client): + base_dir = f'/result/autotest/reports/{date}' + for solution, receiver in solution_receivers.items(): + file_path = f'{base_dir}/{date}/{solution}_daily_report' + smtp_client.send_email(receiver, file_path, f'[{date}] {solution} dockerfile build report') + + +if __name__ == '__main__': + smtp_client = SMTP() + args = sys.argv[1:] + date = '' + if args: + # specify generate date + date = args[0] + if not date: + date = time.strftime('%Y%m%d', time.localtime(time.time())) + + solution_email_receivers = read_solution_email_receiver() + + dispatch_email(date, solution_email_receivers, smtp_client) + + smtp_client.quit() diff --git a/dispatch/smtp.py b/dispatch/smtp.py new file mode 100644 index 0000000..0fc1cd5 --- /dev/null +++ b/dispatch/smtp.py @@ -0,0 +1,60 @@ +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.header import Header +import smtplib + + +class SMTP: + def __init__(self): + with open('/etc/mail.rc', 'r') as fp: + lines = fp.readlines() + self.server = None + self.host = '' + self.port = 65535 + self.user = '' + self.passwd = '' + self.sender = '' + for line in lines: + if line.startswith('set smtp=smtps://'): + self.host = line.split(':')[1][2:] + self.port = int(line.split(':')[-1]) + if line.startswith('set smtp-auth-user='): + self.user = line.split('=')[-1] + if line.startswith('set smtp-auth-password='): + self.passwd = line.split('=')[-1] + if line.startswith('set from='): + self.sender = line.split('=')[-1] + + self.reconnect() + + def reconnect(self): + self.server = smtplib.SMTP_SSL(self.host, self.port) + self.server.login(self.user, self.passwd) + + def send_email(self, receiver, file_path, subject): + with open(file_path, 'r') as fp: + content = fp.read() + message = MIMEMultipart() + message['From'] = Header(self.sender) + message['To'] = Header(receiver) + message['Subject'] = Header(subject) + message.attach(MIMEText(content, 'plain', 'utf-8')) + attachment = MIMEText(content, 'base64', 'utf-8') + attachment['Content-Type'] = 'application/octet-stream' + attachment['Content-Disposition'] = 'attachment; filename="report.txt"' + message.attach(attachment) + return self.send_out(receiver, message) + + def send_out(self, receiver, message): + try: + self.server.sendmail(self.sender, receiver, message.as_string()) + return True + except smtplib.SMTPException as e: + self.reconnect() + try: + self.server.sendmail(self.sender, receiver, message.as_string()) + return True + except smtplib.SMTPException as e: + # retry failed, give up + pass + return False diff --git a/generate/gen_report.py b/generate/gen_report.py index aba8cde..b146cd1 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -1,7 +1,6 @@ #!/usr/bin/python3 # coding: utf-8 -import logging import os import sys import time @@ -18,14 +17,8 @@ def begin(local_date='') -> Tuple[bool, str, str, str]: # construct log and output path for today log_path = f'/result/autotest/actuators/{local_date}/log' output_path = f'/result/autotest/actuators/{local_date}/output' - has_build_today = True # check if there is build today by check log/output path - if not os.path.exists(log_path): - logging.log(logging.WARNING, f'未检测到{local_date}的构建日志目录') - has_build_today = False - if not os.path.exists(output_path): - logging.log(logging.WARNING, f'未检测到{local_date}的构建输出目录') - has_build_today = False + has_build_today = os.path.exists(log_path) and os.path.exists(output_path) return has_build_today, local_date, log_path, output_path -- Gitee From 60530d3235eee158e6a9a0c2d334ab94797f03f4 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 11:14:24 +0800 Subject: [PATCH 38/64] Fix: config parsing error, method error --- dispatch/send_report.py | 2 +- dispatch/smtp.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/dispatch/send_report.py b/dispatch/send_report.py index 12a3ca7..7e4f20e 100644 --- a/dispatch/send_report.py +++ b/dispatch/send_report.py @@ -24,7 +24,7 @@ def read_solution_email_receiver() -> Dict[str, str]: def dispatch_email(date, solution_receivers, smtp_client): base_dir = f'/result/autotest/reports/{date}' for solution, receiver in solution_receivers.items(): - file_path = f'{base_dir}/{date}/{solution}_daily_report' + file_path = f'{base_dir}/{solution}_daily_report' smtp_client.send_email(receiver, file_path, f'[{date}] {solution} dockerfile build report') diff --git a/dispatch/smtp.py b/dispatch/smtp.py index 0fc1cd5..9969a6c 100644 --- a/dispatch/smtp.py +++ b/dispatch/smtp.py @@ -16,14 +16,14 @@ class SMTP: self.sender = '' for line in lines: if line.startswith('set smtp=smtps://'): - self.host = line.split(':')[1][2:] - self.port = int(line.split(':')[-1]) + self.host = line.split(':')[1][2:].strip() + self.port = int(line.split(':')[-1].strip()) if line.startswith('set smtp-auth-user='): - self.user = line.split('=')[-1] + self.user = line.split('=')[-1].strip() if line.startswith('set smtp-auth-password='): - self.passwd = line.split('=')[-1] + self.passwd = line.split('=')[-1].strip() if line.startswith('set from='): - self.sender = line.split('=')[-1] + self.sender = line.split('=')[-1].strip() self.reconnect() @@ -58,3 +58,6 @@ class SMTP: # retry failed, give up pass return False + + def quit(self): + self.server.quit() -- Gitee From 13e6fcbad4cbf49391c98ec5d9f8c0bf8f20b840 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 11:34:05 +0800 Subject: [PATCH 39/64] Update: add configuration instruction for report dispatch --- README.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index beaf2f1..0b0fa25 100644 --- a/README.md +++ b/README.md @@ -5,19 +5,30 @@ 归档日志自动生成报告，发送邮件通知 -#### +#### 日志生成配置方法 1. 克隆此仓库到本地 2. 为服务器URL提示创建配置文件 generate/config.json，如下： ```json { - "address": "example.com/archive_path/" + "address": "http://example.com/archive_path/" } ``` +#### 日志发送配置方法 +1. 克隆此仓库到本地（如已克隆，可忽略） +2. 为解决方案、收件人创建配置文件 dispatch/config.json，如下： +```json +{ + "SolutionName1": "receiver1@example.com", + "SolutionName2": "receiver2@example.com", + "SolutionName3": "receiver3@example.com" +} +``` + #### 使用教程 1. 执行 generate/gen_report.py 以生成报告等 -2. 执行 xxxx 以发送邮件通知 +2. 执行 dispatch/send_report.py 以发送邮件通知 -- Gitee From b1d524c62f6672076ae96e4ad6e1dabb49b01c2c Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 15:01:48 +0800 Subject: [PATCH 40/64] Enhancement: try html report send to maintainer --- dispatch/smtp.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dispatch/smtp.py b/dispatch/smtp.py index 9969a6c..f4692e7 100644 --- a/dispatch/smtp.py +++ b/dispatch/smtp.py @@ -1,3 +1,4 @@ +from ansi2html import Ansi2HTMLConverter from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.header import Header @@ -25,6 +26,7 @@ class SMTP: if line.startswith('set from='): self.sender = line.split('=')[-1].strip() + self.conv = Ansi2HTMLConverter() self.reconnect() def reconnect(self): @@ -38,10 +40,11 @@ class SMTP: message['From'] = Header(self.sender) message['To'] = Header(receiver) message['Subject'] = Header(subject) - message.attach(MIMEText(content, 'plain', 'utf-8')) - attachment = MIMEText(content, 'base64', 'utf-8') + html = self.conv.convert(content) + message.attach(MIMEText(html, 'html', 'utf-8')) + attachment = MIMEText(html, 'base64', 'utf-8') attachment['Content-Type'] = 'application/octet-stream' - attachment['Content-Disposition'] = 'attachment; filename="report.txt"' + attachment['Content-Disposition'] = 'attachment; filename="report.html"' message.attach(attachment) return self.send_out(receiver, message) -- Gitee From 614395a00e4863e7dff9f54c5580837576b2c113 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 15:10:16 +0800 Subject: [PATCH 41/64] Enhancement: enhance report output --- generate/maintask.py | 6 +++--- generate/subtask.py | 4 +++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/generate/maintask.py b/generate/maintask.py index 3a45b99..a5aaffa 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -35,7 +35,7 @@ class Maintask: self.subtasks = [] self.success = 0 self.fail = 0 - self.result = 'PASS' + self.result = '\033[1;32mPASS\033[0m' def __str__(self): passrate = "{:.1f}".format(self.success / (self.success + self.fail) * 100) @@ -53,7 +53,7 @@ Subtask(s) build Status: def subtasks_info_general(self): sub_task_general = sorted([subtask.general_status() for subtask in self.subtasks]) - sub_task_general = ['\tProducer\tImage Name\t\t\t\tBuild Status'] + sub_task_general + sub_task_general = [f'\t{"Producer":16s}{"Image Name":42s}Build Status'] + sub_task_general return '\n'.join(sub_task_general) def subtasks_info_detail(self): @@ -64,5 +64,5 @@ Subtask(s) build Status: self.success += 1 else: self.fail += 1 - self.result = '**FAIL**' + self.result = '\033[1;31mFAIL**\033[0m' self.subtasks.append(sub_task) diff --git a/generate/subtask.py b/generate/subtask.py index 18ba33f..e130c2f 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -41,7 +41,9 @@ class Subtask: def detail_status(self): newline = '\n' - return f"""{self.image}: {'**FAIL**' if self.status != 'PASS' else 'PASS'} + fail_output = '\033[1;31mFAIL\033[0m' + pass_output = '\033[1;32mPASS\033[0m' + return f"""{self.image}: {fail_output if self.status != 'PASS' else pass_output} Producer: {self.producer} Log: {self.log} Time: {self.start_time} ~ {self.end_time}, Cost {self.duration} -- Gitee From be2414d83c0a9172dc84405ef1d756e16cb5942d Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 15:16:18 +0800 Subject: [PATCH 42/64] Fix: incorrect output enhancement --- generate/maintask.py | 4 ++-- generate/subtask.py | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/generate/maintask.py b/generate/maintask.py index a5aaffa..9615d73 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -35,7 +35,7 @@ class Maintask: self.subtasks = [] self.success = 0 self.fail = 0 - self.result = '\033[1;32mPASS\033[0m' + self.result = '\033[32m PASS \033[0m' def __str__(self): passrate = "{:.1f}".format(self.success / (self.success + self.fail) * 100) @@ -64,5 +64,5 @@ Subtask(s) build Status: self.success += 1 else: self.fail += 1 - self.result = '\033[1;31mFAIL**\033[0m' + self.result = '\033[31m FAIL \033[0m' self.subtasks.append(sub_task) diff --git a/generate/subtask.py b/generate/subtask.py index e130c2f..460fe49 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -37,13 +37,15 @@ class Subtask: return f'{self.image}: {self.output}\n' def general_status(self): - return f'\t{self.producer:16s}{self.image:42s}{"**FAIL**" if self.status != "PASS" else "PASS"}' + fail_output = '\033[31m FAIL \033[0m' + pass_output = '\033[32m PASS \033[0m' + return f'\t{self.producer:16s}{self.image:42s}{fail_output if self.status != "PASS" else pass_output}' def detail_status(self): newline = '\n' - fail_output = '\033[1;31mFAIL\033[0m' - pass_output = '\033[1;32mPASS\033[0m' - return f"""{self.image}: {fail_output if self.status != 'PASS' else pass_output} + fail_output = '\033[31m FAIL \033[0m' + pass_output = '\033[32m PASS \033[0m' + return f"""{self.image}:{fail_output if self.status != 'PASS' else pass_output} Producer: {self.producer} Log: {self.log} Time: {self.start_time} ~ {self.end_time}, Cost {self.duration} -- Gitee From 9ce586a237b997b469cbc99e8a74978de6b43ea1 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 15:48:30 +0800 Subject: [PATCH 43/64] Enhancement: use html report for all, enhance output --- dispatch/send_report.py | 2 +- dispatch/smtp.py | 8 ++------ generate/gen_report.py | 21 ++++++++++++--------- generate/maintask.py | 19 +++++++++++-------- generate/subtask.py | 3 ++- 5 files changed, 28 insertions(+), 25 deletions(-) diff --git a/dispatch/send_report.py b/dispatch/send_report.py index 7e4f20e..a513e16 100644 --- a/dispatch/send_report.py +++ b/dispatch/send_report.py @@ -24,7 +24,7 @@ def read_solution_email_receiver() -> Dict[str, str]: def dispatch_email(date, solution_receivers, smtp_client): base_dir = f'/result/autotest/reports/{date}' for solution, receiver in solution_receivers.items(): - file_path = f'{base_dir}/{solution}_daily_report' + file_path = f'{base_dir}/{solution}_daily_report.html' smtp_client.send_email(receiver, file_path, f'[{date}] {solution} dockerfile build report') diff --git a/dispatch/smtp.py b/dispatch/smtp.py index f4692e7..a287814 100644 --- a/dispatch/smtp.py +++ b/dispatch/smtp.py @@ -1,4 +1,3 @@ -from ansi2html import Ansi2HTMLConverter from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.header import Header @@ -25,8 +24,6 @@ class SMTP: self.passwd = line.split('=')[-1].strip() if line.startswith('set from='): self.sender = line.split('=')[-1].strip() - - self.conv = Ansi2HTMLConverter() self.reconnect() def reconnect(self): @@ -40,9 +37,8 @@ class SMTP: message['From'] = Header(self.sender) message['To'] = Header(receiver) message['Subject'] = Header(subject) - html = self.conv.convert(content) - message.attach(MIMEText(html, 'html', 'utf-8')) - attachment = MIMEText(html, 'base64', 'utf-8') + message.attach(MIMEText(content, 'html', 'utf-8')) + attachment = MIMEText(content, 'base64', 'utf-8') attachment['Content-Type'] = 'application/octet-stream' attachment['Content-Disposition'] = 'attachment; filename="report.html"' message.attach(attachment) diff --git a/generate/gen_report.py b/generate/gen_report.py index b146cd1..453d100 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 # coding: utf-8 +from ansi2html import Ansi2HTMLConverter import os import sys import time @@ -35,7 +36,7 @@ def find_log_files( return main_logs, subtask_logs -def read_main_log(main_logs: List[str]) -> List[Maintask]: +def read_main_log(main_logs: List[str], output_path: str) -> List[Maintask]: maintasks = [] for log_file in main_logs: with open(log_file, 'r') as fp: @@ -70,7 +71,8 @@ def read_main_log(main_logs: List[str]) -> List[Maintask]: start_time = ' '.join(line.split()[-2:]).strip() if 'Build task end time' in line: end_time = ' '.join(line.split()[-2:]).strip() - cur_task = Maintask(task, log_file, build_os, arch, docker, solution, producer, version, start_time, end_time) + output = f'{output_path}/{solution}_daily_output' + cur_task = Maintask(task, log_file, build_os, arch, docker, solution, producer, version, start_time, end_time, output) maintasks.append(cur_task) return maintasks @@ -146,8 +148,8 @@ def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: for task in sol_main_tasks: report_infos_brief.append(str(task) + task.subtasks_info_general()) report_infos_detail.append(task.subtasks_info_detail()) - solution_reports[solution] = '\n'.join(report_infos_brief) \ - + f'\n\n{"=" * 120}\n\t[Subtask Detail Report]\n\n' \ + solution_reports[solution] = '\n'.join(sorted(report_infos_brief)) \ + + f'\n\n{"=" * 120}\n\n\t\033[1m [Subtask Detail Report] \033[0m\n\n' \ + '\n'.join(report_infos_detail) return solution_reports @@ -173,14 +175,15 @@ def generate_output(main_tasks: List[Maintask]) -> Dict[str, str]: def write_reports( local_date: str, reports: Dict[str, str], report_path: str) -> None: + converter = Ansi2HTMLConverter() if not os.path.exists(report_path): os.makedirs(report_path) for solution, report in reports.items(): - file_path = f'{report_path}/{solution}_daily_report' + file_path = f'{report_path}/{solution}_daily_report.html' with open(file_path, 'w') as fp: - fp.write( - f'\t\t{solution} {local_date} Build Report\n' + - report) + content = f'\t\t\033[1m[{solution} {local_date} Build Brief Report]\033[0m\n' + report + content_html = converter.convert(content) + fp.write(content_html) def write_outputs( @@ -209,7 +212,7 @@ if __name__ == '__main__': # find available log files main_logs, subtask_logs = find_log_files(local_date, log_path) - main_tasks = read_main_log(main_logs) + main_tasks = read_main_log(main_logs, '/result/autotest/dc-build-latest') # read build status and output information main_tasks = read_subtask_log(subtask_logs, main_tasks, local_date) diff --git a/generate/maintask.py b/generate/maintask.py index 9615d73..bfb1fc8 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -16,8 +16,8 @@ class Maintask: # no address provided, use local server = "/" - def __init__(self, task, log, curr_os, arch, docker, solution, producer, version, start_time, end_time): - self.task = task + def __init__(self, task, log, curr_os, arch, docker, solution, producer, version, start_time, end_time, output): + self.task = task.split('/')[2] self.log = self.server + '/'.join(log.split('/')[3:]) self.log_dir = os.path.dirname(log) self.os = curr_os @@ -28,6 +28,7 @@ class Maintask: self.version = version self.start_time = start_time self.end_time = end_time + self.output = self.server + '/'.join(output.split('/')[2:]) start = datetime.strptime(self.start_time, '%Y-%m-%d %H:%M:%S') end = datetime.strptime(self.end_time, '%Y-%m-%d %H:%M:%S') delta = end - start @@ -39,16 +40,18 @@ class Maintask: def __str__(self): passrate = "{:.1f}".format(self.success / (self.success + self.fail) * 100) - return f"""\n{'=' * 80} -Task Name: {self.task} + return f"""\n +Producer: {self.task} Version: {self.version} -Logfile: {self.log} OS: {self.os} Architecture: {self.arch} Docker version: {self.docker} -Build time: {self.start_time} ~ {self.end_time}, Cost {self.duration} -Result: {self.result}. Subtasks: {self.success} PASS {self.fail} FAIL; PASS RATE {passrate}% -Subtask(s) build Status: +Result: {self.result}. PASS rate {passrate}%. +Build time: {self.start_time} ~ {self.end_time} +Cost: {self.duration} +Logfile: {self.log} +Output: {self.output} +Subtasks: {self.success} PASS; {self.fail} FAIL: """ def subtasks_info_general(self): diff --git a/generate/subtask.py b/generate/subtask.py index 460fe49..aa39c06 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -48,6 +48,7 @@ class Subtask: return f"""{self.image}:{fail_output if self.status != 'PASS' else pass_output} Producer: {self.producer} Log: {self.log} -Time: {self.start_time} ~ {self.end_time}, Cost {self.duration} +Time: {self.start_time} ~ {self.end_time} +Cost: {self.duration} {newline.join(('Help message: (last 10 lines of log)', self.error)) if self.status != 'PASS' else ''} """ -- Gitee From 4b80ee9386caada03349c3affbcaae581ba2d52b Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 15:52:44 +0800 Subject: [PATCH 44/64] Fix: put solution name there if image name is too short --- generate/maintask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/maintask.py b/generate/maintask.py index bfb1fc8..fd3c32c 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -17,7 +17,7 @@ class Maintask: server = "/" def __init__(self, task, log, curr_os, arch, docker, solution, producer, version, start_time, end_time, output): - self.task = task.split('/')[2] + self.task = task.split('/')[2] if len(task.split('/')) > 2 else solution self.log = self.server + '/'.join(log.split('/')[3:]) self.log_dir = os.path.dirname(log) self.os = curr_os -- Gitee From 427e2ac31b42346bf5080e9d5cc59bc0ef9d3d8b Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 16:00:18 +0800 Subject: [PATCH 45/64] Enhancement: enhance output format --- generate/gen_report.py | 10 ++++++---- generate/maintask.py | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 453d100..3d9e03e 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -72,7 +72,8 @@ def read_main_log(main_logs: List[str], output_path: str) -> List[Maintask]: if 'Build task end time' in line: end_time = ' '.join(line.split()[-2:]).strip() output = f'{output_path}/{solution}_daily_output' - cur_task = Maintask(task, log_file, build_os, arch, docker, solution, producer, version, start_time, end_time, output) + cur_task = Maintask(task, log_file, build_os, arch, docker, solution, producer, version, start_time, end_time, + output) maintasks.append(cur_task) return maintasks @@ -149,8 +150,9 @@ def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: report_infos_brief.append(str(task) + task.subtasks_info_general()) report_infos_detail.append(task.subtasks_info_detail()) solution_reports[solution] = '\n'.join(sorted(report_infos_brief)) \ - + f'\n\n{"=" * 120}\n\n\t\033[1m [Subtask Detail Report] \033[0m\n\n' \ - + '\n'.join(report_infos_detail) + + f'\n\n{"=" * 160}\n\n\n' \ + + '\t\t\033[1m[Subtask Detail Report]\033[0m\n\n\n' \ + + '\n'.join(report_infos_detail) return solution_reports @@ -165,7 +167,7 @@ def generate_output(main_tasks: List[Maintask]) -> Dict[str, str]: infos = [] for task in main_task.subtasks: infos.append(task.output_info()) - output_infos.append('=' * 80 + '\n') + output_infos.append('\n' + '=' * 80 + '\n') output_infos.append(f'Producer: {main_task.producer}\n') output_infos.extend(infos) solution_outputs[solution] = ''.join(output_infos) diff --git a/generate/maintask.py b/generate/maintask.py index fd3c32c..2d9cf9a 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -28,7 +28,7 @@ class Maintask: self.version = version self.start_time = start_time self.end_time = end_time - self.output = self.server + '/'.join(output.split('/')[2:]) + self.output = self.server + '/'.join(output.split('/')[3:]) start = datetime.strptime(self.start_time, '%Y-%m-%d %H:%M:%S') end = datetime.strptime(self.end_time, '%Y-%m-%d %H:%M:%S') delta = end - start -- Gitee From 5df411df63569b1c5d5b2aba9e1615aa5d68646c Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 16:09:29 +0800 Subject: [PATCH 46/64] Fix: sort subtasks order in report --- generate/gen_report.py | 1 + generate/maintask.py | 1 + generate/subtask.py | 4 +++- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 3d9e03e..6e5eef8 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -120,6 +120,7 @@ def read_subtask_log(subtask_logs: List[str], main_tasks: List[Maintask], local_ image_name, main_task.solution, main_task.producer, + main_task.version, subtask_result, subtask_output, subtask_error, diff --git a/generate/maintask.py b/generate/maintask.py index 2d9cf9a..0d49ecb 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -60,6 +60,7 @@ Subtasks: {self.success} PASS; {self.fail} FAIL: return '\n'.join(sub_task_general) def subtasks_info_detail(self): + self.subtasks.sort(key=lambda x: (x.producer, x.version, x.image)) return '\n'.join(sorted([subtask.detail_status() for subtask in self.subtasks])) def add_subtask(self, sub_task: Subtask): diff --git a/generate/subtask.py b/generate/subtask.py index aa39c06..ff192cd 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -13,13 +13,14 @@ class Subtask: # no address provided, use local server = "/" - def __init__(self, date, log, image, solution, producer, + def __init__(self, date, log, image, solution, producer, version, status, output, error, start_time, end_time): self.date = date self.log = self.server + '/'.join(log.split('/')[3:]) self.image = image self.solution = solution self.producer = producer + self.version = version self.status = status self.output = output if not self.output.startswith("Not Available"): @@ -47,6 +48,7 @@ class Subtask: pass_output = '\033[32m PASS \033[0m' return f"""{self.image}:{fail_output if self.status != 'PASS' else pass_output} Producer: {self.producer} +Version: {self.version} Log: {self.log} Time: {self.start_time} ~ {self.end_time} Cost: {self.duration} -- Gitee From 21117d17f660de388bd47b79a7822fba13ced6a9 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 16:31:35 +0800 Subject: [PATCH 47/64] Enhancement: Formatting files, sorting subtask logs --- dispatch/send_report.py | 5 ++++- dispatch/smtp.py | 5 +++-- generate/gen_report.py | 3 ++- generate/maintask.py | 22 +++++++++++++++------- generate/subtask.py | 5 +---- 5 files changed, 25 insertions(+), 15 deletions(-) diff --git a/dispatch/send_report.py b/dispatch/send_report.py index a513e16..a20da72 100644 --- a/dispatch/send_report.py +++ b/dispatch/send_report.py @@ -25,7 +25,10 @@ def dispatch_email(date, solution_receivers, smtp_client): base_dir = f'/result/autotest/reports/{date}' for solution, receiver in solution_receivers.items(): file_path = f'{base_dir}/{solution}_daily_report.html' - smtp_client.send_email(receiver, file_path, f'[{date}] {solution} dockerfile build report') + smtp_client.send_email( + receiver, + file_path, + f'[{date}] {solution} dockerfile build report') if __name__ == '__main__': diff --git a/dispatch/smtp.py b/dispatch/smtp.py index a287814..e63b84d 100644 --- a/dispatch/smtp.py +++ b/dispatch/smtp.py @@ -37,7 +37,7 @@ class SMTP: message['From'] = Header(self.sender) message['To'] = Header(receiver) message['Subject'] = Header(subject) - message.attach(MIMEText(content, 'html', 'utf-8')) + message.attach(MIMEText(content, 'html', 'utf-8')) attachment = MIMEText(content, 'base64', 'utf-8') attachment['Content-Type'] = 'application/octet-stream' attachment['Content-Disposition'] = 'attachment; filename="report.html"' @@ -51,7 +51,8 @@ class SMTP: except smtplib.SMTPException as e: self.reconnect() try: - self.server.sendmail(self.sender, receiver, message.as_string()) + self.server.sendmail( + self.sender, receiver, message.as_string()) return True except smtplib.SMTPException as e: # retry failed, give up diff --git a/generate/gen_report.py b/generate/gen_report.py index 6e5eef8..4b85011 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -79,7 +79,8 @@ def read_main_log(main_logs: List[str], output_path: str) -> List[Maintask]: return maintasks -def read_subtask_log(subtask_logs: List[str], main_tasks: List[Maintask], local_date: str) -> List[Maintask]: +def read_subtask_log( + subtask_logs: List[str], main_tasks: List[Maintask], local_date: str) -> List[Maintask]: # check subtask logs by solution and producer for main_task in main_tasks: for subtask_log in subtask_logs: diff --git a/generate/maintask.py b/generate/maintask.py index 0d49ecb..ad39b18 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -16,8 +16,10 @@ class Maintask: # no address provided, use local server = "/" - def __init__(self, task, log, curr_os, arch, docker, solution, producer, version, start_time, end_time, output): - self.task = task.split('/')[2] if len(task.split('/')) > 2 else solution + def __init__(self, task, log, curr_os, arch, docker, solution, + producer, version, start_time, end_time, output): + self.task = task.split( + '/')[2] if len(task.split('/')) > 2 else solution self.log = self.server + '/'.join(log.split('/')[3:]) self.log_dir = os.path.dirname(log) self.os = curr_os @@ -39,7 +41,8 @@ class Maintask: self.result = '\033[32m PASS \033[0m' def __str__(self): - passrate = "{:.1f}".format(self.success / (self.success + self.fail) * 100) + passrate = "{:.1f}".format( + self.success / (self.success + self.fail) * 100) return f"""\n Producer: {self.task} Version: {self.version} @@ -55,13 +58,18 @@ Subtasks: {self.success} PASS; {self.fail} FAIL: """ def subtasks_info_general(self): - sub_task_general = sorted([subtask.general_status() for subtask in self.subtasks]) - sub_task_general = [f'\t{"Producer":16s}{"Image Name":42s}Build Status'] + sub_task_general + sub_task_general = sorted([subtask.general_status() + for subtask in self.subtasks]) + sub_task_general = [ + f'\t{"Producer":16s}{"Image Name":42s}Build Status'] + sub_task_general return '\n'.join(sub_task_general) def subtasks_info_detail(self): - self.subtasks.sort(key=lambda x: (x.producer, x.version, x.image)) - return '\n'.join(sorted([subtask.detail_status() for subtask in self.subtasks])) + sub_task_detail = sorted([subtask.detail_status() + for subtask in self.subtasks]) + sub_task_detail = [ + f'\tProducer: {self.producer}, Version: {self.version}\n\n'] + sub_task_detail + return '\n'.join(sub_task_detail) def add_subtask(self, sub_task: Subtask): if sub_task.status == 'PASS': diff --git a/generate/subtask.py b/generate/subtask.py index ff192cd..9182f91 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -47,10 +47,7 @@ class Subtask: fail_output = '\033[31m FAIL \033[0m' pass_output = '\033[32m PASS \033[0m' return f"""{self.image}:{fail_output if self.status != 'PASS' else pass_output} -Producer: {self.producer} -Version: {self.version} Log: {self.log} Time: {self.start_time} ~ {self.end_time} Cost: {self.duration} -{newline.join(('Help message: (last 10 lines of log)', self.error)) if self.status != 'PASS' else ''} -""" +{newline.join(('Help message: (last 10 lines of log)', self.error)) if self.status != 'PASS' else ''}""" -- Gitee From d20fb7556c75815d0e4ee11ce87a5896d517bdb4 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Fri, 4 Dec 2020 16:35:24 +0800 Subject: [PATCH 48/64] Fix: subtask sorting - should be producer - version - image --- generate/gen_report.py | 2 +- generate/maintask.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 4b85011..662b3d6 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -154,7 +154,7 @@ def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: solution_reports[solution] = '\n'.join(sorted(report_infos_brief)) \ + f'\n\n{"=" * 160}\n\n\n' \ + '\t\t\033[1m[Subtask Detail Report]\033[0m\n\n\n' \ - + '\n'.join(report_infos_detail) + + '\n'.join(sorted(report_infos_detail)) return solution_reports diff --git a/generate/maintask.py b/generate/maintask.py index ad39b18..66e2839 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -60,15 +60,16 @@ Subtasks: {self.success} PASS; {self.fail} FAIL: def subtasks_info_general(self): sub_task_general = sorted([subtask.general_status() for subtask in self.subtasks]) - sub_task_general = [ - f'\t{"Producer":16s}{"Image Name":42s}Build Status'] + sub_task_general + sub_task_general = [f'\t{"Producer":16s}{"Image Name":42s}Build Status'] \ + + sub_task_general return '\n'.join(sub_task_general) def subtasks_info_detail(self): sub_task_detail = sorted([subtask.detail_status() for subtask in self.subtasks]) - sub_task_detail = [ - f'\tProducer: {self.producer}, Version: {self.version}\n\n'] + sub_task_detail + sub_task_detail = [f'\tProducer: {self.producer}, Version: {self.version}\n'] \ + + sub_task_detail \ + + ['\n'] return '\n'.join(sub_task_detail) def add_subtask(self, sub_task: Subtask): -- Gitee From 3b88bf1e34c847d9f970f2f8661bbbaba16de58f Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 09:49:51 +0800 Subject: [PATCH 49/64] Add: service, main/cc receiver for email --- README.md | 36 ++++++++++++++++++++++++++++++++---- dispatch/send_report.py | 5 ++++- dispatch/smtp.py | 5 +++-- generate/gen_report.py | 27 ++++++++++++++++++++------- generate/maintask.py | 13 +++++++------ generate/subtask.py | 11 ++++++----- 6 files changed, 72 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 0b0fa25..0c74e03 100644 --- a/README.md +++ b/README.md @@ -18,12 +18,40 @@ #### 日志发送配置方法 1. 克隆此仓库到本地（如已克隆，可忽略） -2. 为解决方案、收件人创建配置文件 dispatch/config.json，如下： +2. 修改reportsend.service与report_generate_dispatch.sh，使绝对路径正确匹配 +3. 为解决方案、收件人创建配置文件 dispatch/config.json，如下： ```json { - "SolutionName1": "receiver1@example.com", - "SolutionName2": "receiver2@example.com", - "SolutionName3": "receiver3@example.com" + "SolutionName1": { + "receiver":[ + "receiver1@example.com", + "receiver2@example.com + ], + "cc_receiver":[ + "cc1@example.com", + "cc2@example.com" + ] + }, + "SolutionName2": { + "receiver":[ + "receiver1@example.com", + "receiver2@example.com + ], + "cc_receiver":[ + "cc1@example.com", + "cc2@example.com" + ] + }, + "SolutionName3": { + "receiver":[ + "receiver1@example.com", + "receiver2@example.com + ], + "cc_receiver":[ + "cc1@example.com", + "cc2@example.com" + ] + } } ``` diff --git a/dispatch/send_report.py b/dispatch/send_report.py index a20da72..8e1380d 100644 --- a/dispatch/send_report.py +++ b/dispatch/send_report.py @@ -23,10 +23,13 @@ def read_solution_email_receiver() -> Dict[str, str]: def dispatch_email(date, solution_receivers, smtp_client): base_dir = f'/result/autotest/reports/{date}' - for solution, receiver in solution_receivers.items(): + for solution, receivers in solution_receivers.items(): file_path = f'{base_dir}/{solution}_daily_report.html' + receiver = receivers.get('receiver') + cc_receiver = receivers.get('cc_receiver') smtp_client.send_email( receiver, + cc_receiver, file_path, f'[{date}] {solution} dockerfile build report') diff --git a/dispatch/smtp.py b/dispatch/smtp.py index e63b84d..755fb57 100644 --- a/dispatch/smtp.py +++ b/dispatch/smtp.py @@ -30,12 +30,13 @@ class SMTP: self.server = smtplib.SMTP_SSL(self.host, self.port) self.server.login(self.user, self.passwd) - def send_email(self, receiver, file_path, subject): + def send_email(self, receiver, cc_receiver, file_path, subject): with open(file_path, 'r') as fp: content = fp.read() message = MIMEMultipart() message['From'] = Header(self.sender) - message['To'] = Header(receiver) + message['To'] = Header(';'.join(receiver)) + message['Cc'] = Header(';'.join(cc_receiver)) message['Subject'] = Header(subject) message.attach(MIMEText(content, 'html', 'utf-8')) attachment = MIMEText(content, 'base64', 'utf-8') diff --git a/generate/gen_report.py b/generate/gen_report.py index 662b3d6..7b09a68 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -72,15 +72,25 @@ def read_main_log(main_logs: List[str], output_path: str) -> List[Maintask]: if 'Build task end time' in line: end_time = ' '.join(line.split()[-2:]).strip() output = f'{output_path}/{solution}_daily_output' - cur_task = Maintask(task, log_file, build_os, arch, docker, solution, producer, version, start_time, end_time, + cur_task = Maintask(task, + log_file, + build_os, + arch, + docker, + solution, + producer, + version, + start_time, + end_time, output) maintasks.append(cur_task) return maintasks -def read_subtask_log( - subtask_logs: List[str], main_tasks: List[Maintask], local_date: str) -> List[Maintask]: +def read_subtask_log(subtask_logs: List[str], + main_tasks: List[Maintask], + local_date: str) -> List[Maintask]: # check subtask logs by solution and producer for main_task in main_tasks: for subtask_log in subtask_logs: @@ -90,7 +100,7 @@ def read_subtask_log( lines = fp.readlines() # read lines for information image_name = 'unknown' - subtask_result = 'UNKNOWN' + subtask_result = False subtask_output = 'unknown' subtask_error = '' subtask_start = '1970-01-01 00:00:00' @@ -101,13 +111,14 @@ def read_subtask_log( if "Build subtask output dir:" in line: subtask_output = line.split()[-1] if "Build subtask result:" in line: - subtask_result = line.split()[-1] + subtask_result = True if line.split( + )[-1] == 'PASS' else False if 'Build subtask start time' in line: subtask_start = ' '.join(line.split()[-2:]).strip() if 'Build subtask end time' in line: subtask_end = ' '.join(line.split()[-2:]).strip() - if subtask_result == 'FAIL': + if not subtask_result: # if fail, extract last 10 lines of log for debug subtask_output = "Not Available due to FAIL" length = len(lines) @@ -185,7 +196,9 @@ def write_reports( for solution, report in reports.items(): file_path = f'{report_path}/{solution}_daily_report.html' with open(file_path, 'w') as fp: - content = f'\t\t\033[1m[{solution} {local_date} Build Brief Report]\033[0m\n' + report + content = f'\t\t\033[1m[{solution} {local_date}' \ + + 'Build Brief Report]\033[0m\n' \ + + report content_html = converter.convert(content) fp.write(content_html) diff --git a/generate/maintask.py b/generate/maintask.py index 66e2839..605ddee 100644 --- a/generate/maintask.py +++ b/generate/maintask.py @@ -60,20 +60,21 @@ Subtasks: {self.success} PASS; {self.fail} FAIL: def subtasks_info_general(self): sub_task_general = sorted([subtask.general_status() for subtask in self.subtasks]) - sub_task_general = [f'\t{"Producer":16s}{"Image Name":42s}Build Status'] \ - + sub_task_general + sub_task_general.insert(0, + f'\t{"Producer":16s}{"Image Name":42s}Build Status') return '\n'.join(sub_task_general) def subtasks_info_detail(self): sub_task_detail = sorted([subtask.detail_status() for subtask in self.subtasks]) - sub_task_detail = [f'\tProducer: {self.producer}, Version: {self.version}\n'] \ - + sub_task_detail \ - + ['\n'] + sub_task_detail = [f'\t\033[1mProducer: {self.producer}, ' + f'Version: {self.version}\033[0m\n'] \ + + sub_task_detail \ + + ['\n'] return '\n'.join(sub_task_detail) def add_subtask(self, sub_task: Subtask): - if sub_task.status == 'PASS': + if sub_task.status: self.success += 1 else: self.fail += 1 diff --git a/generate/subtask.py b/generate/subtask.py index 9182f91..6d096ab 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -40,14 +40,15 @@ class Subtask: def general_status(self): fail_output = '\033[31m FAIL \033[0m' pass_output = '\033[32m PASS \033[0m' - return f'\t{self.producer:16s}{self.image:42s}{fail_output if self.status != "PASS" else pass_output}' + return f'\t{self.producer:16s}{self.image:42s}' \ + + f'{fail_output if self.status else pass_output}' def detail_status(self): newline = '\n' - fail_output = '\033[31m FAIL \033[0m' - pass_output = '\033[32m PASS \033[0m' - return f"""{self.image}:{fail_output if self.status != 'PASS' else pass_output} + fail_out = '\033[31m FAIL \033[0m' + pass_out = '\033[32m PASS \033[0m' + return f"""{self.image}:{pass_out if self.status else fail_out} Log: {self.log} Time: {self.start_time} ~ {self.end_time} Cost: {self.duration} -{newline.join(('Help message: (last 10 lines of log)', self.error)) if self.status != 'PASS' else ''}""" +{newline.join(('Tail of log:', self.error)) if self.status else ''}""" -- Gitee From 4d31642f1730e30bc10536d77faf0153bb90aa44 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 09:56:39 +0800 Subject: [PATCH 50/64] Add: sample service file for report send --- report_generate_dispatch.sh | 5 +++++ reportsend.service | 11 +++++++++++ 2 files changed, 16 insertions(+) create mode 100644 report_generate_dispatch.sh create mode 100644 reportsend.service diff --git a/report_generate_dispatch.sh b/report_generate_dispatch.sh new file mode 100644 index 0000000..1513dce --- /dev/null +++ b/report_generate_dispatch.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +python3 generate/gen_report.py +python3 dispatch/send_report.py + diff --git a/reportsend.service b/reportsend.service new file mode 100644 index 0000000..2518d2a --- /dev/null +++ b/reportsend.service @@ -0,0 +1,11 @@ +[Unit] +Description=ReportGenerateAndSend + +[Service] +ExecStart=/bin/bash /root/report/report_generate_dispatch.sh + +[Timer] +OnUnitActiveSec=*-*-* 11:50:00 + +[Install] +WantedBy=multi-user.target -- Gitee From b0e3e10641778dc0940c7245076f61257f847190 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 10:34:55 +0800 Subject: [PATCH 51/64] Fix: smtp msg cc, in new python3 way Signed-off-by: XZhouQD --- dispatch/smtp.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/dispatch/smtp.py b/dispatch/smtp.py index 755fb57..96ac6eb 100644 --- a/dispatch/smtp.py +++ b/dispatch/smtp.py @@ -8,7 +8,7 @@ class SMTP: def __init__(self): with open('/etc/mail.rc', 'r') as fp: lines = fp.readlines() - self.server = None + self.host = '' self.port = 65535 self.user = '' @@ -24,20 +24,20 @@ class SMTP: self.passwd = line.split('=')[-1].strip() if line.startswith('set from='): self.sender = line.split('=')[-1].strip() + self.server = smtplib.SMTP_SSL(self.host, self.port) self.reconnect() def reconnect(self): - self.server = smtplib.SMTP_SSL(self.host, self.port) self.server.login(self.user, self.passwd) def send_email(self, receiver, cc_receiver, file_path, subject): with open(file_path, 'r') as fp: content = fp.read() message = MIMEMultipart() - message['From'] = Header(self.sender) - message['To'] = Header(';'.join(receiver)) - message['Cc'] = Header(';'.join(cc_receiver)) - message['Subject'] = Header(subject) + message['From'] = self.sender + message['To'] = ','.join(receiver) + message['CC'] = ','.join(cc_receiver) + message['Subject'] = subject message.attach(MIMEText(content, 'html', 'utf-8')) attachment = MIMEText(content, 'base64', 'utf-8') attachment['Content-Type'] = 'application/octet-stream' @@ -47,13 +47,12 @@ class SMTP: def send_out(self, receiver, message): try: - self.server.sendmail(self.sender, receiver, message.as_string()) + self.server.send_message(message) return True except smtplib.SMTPException as e: self.reconnect() try: - self.server.sendmail( - self.sender, receiver, message.as_string()) + self.server.send_message(message) return True except smtplib.SMTPException as e: # retry failed, give up -- Gitee From 10f280a884752822582c5a56edc09abcf7805ce7 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 10:50:40 +0800 Subject: [PATCH 52/64] Add: Timer service Signed-off-by: XZhouQD --- README.md | 5 +++-- reportsend.service | 6 ------ reportsend.timer | 9 +++++++++ 3 files changed, 12 insertions(+), 8 deletions(-) create mode 100644 reportsend.timer diff --git a/README.md b/README.md index 0c74e03..9ae3716 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,9 @@ #### 日志发送配置方法 1. 克隆此仓库到本地（如已克隆，可忽略） -2. 修改reportsend.service与report_generate_dispatch.sh，使绝对路径正确匹配 -3. 为解决方案、收件人创建配置文件 dispatch/config.json，如下： +2. 修改reportsend.service与report_generate_dispatch.sh，使绝对路径正确匹配。修改reportsend.timer设置定时发送时间 +3. 复制reportsend.service与reportsend.timer到/usr/lib/systemd/system/内，启动/设置自启动此timer +4. 为解决方案、收件人创建配置文件 dispatch/config.json，如下： ```json { "SolutionName1": { diff --git a/reportsend.service b/reportsend.service index 2518d2a..aa7fc4e 100644 --- a/reportsend.service +++ b/reportsend.service @@ -3,9 +3,3 @@ Description=ReportGenerateAndSend [Service] ExecStart=/bin/bash /root/report/report_generate_dispatch.sh - -[Timer] -OnUnitActiveSec=*-*-* 11:50:00 - -[Install] -WantedBy=multi-user.target diff --git a/reportsend.timer b/reportsend.timer new file mode 100644 index 0000000..6120280 --- /dev/null +++ b/reportsend.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Runs reportsend every day + +[Timer] +OnUnitActiveSec=*-*-* 11:50:00 +Unit=reportsend.service + +[Install] +WantedBy=multi-user.target \ No newline at end of file -- Gitee From 80417433728b50a5c1af0f8dc3ee046db3d048a3 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 10:54:50 +0800 Subject: [PATCH 53/64] Fix: Timer should be run onCalendar Signed-off-by: XZhouQD --- reportsend.timer | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reportsend.timer b/reportsend.timer index 6120280..69f58c6 100644 --- a/reportsend.timer +++ b/reportsend.timer @@ -2,7 +2,7 @@ Description=Runs reportsend every day [Timer] -OnUnitActiveSec=*-*-* 11:50:00 +OnCalendar=*-*-* 11:50:00 Unit=reportsend.service [Install] -- Gitee From 092488c3cbbdc008efdb3385806fd89f70e4553b Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 10:55:44 +0800 Subject: [PATCH 54/64] Fix: use absolute path in script Signed-off-by: XZhouQD --- report_generate_dispatch.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/report_generate_dispatch.sh b/report_generate_dispatch.sh index 1513dce..4e75ea9 100644 --- a/report_generate_dispatch.sh +++ b/report_generate_dispatch.sh @@ -1,5 +1,5 @@ #!/bin/bash -python3 generate/gen_report.py -python3 dispatch/send_report.py +python3 /root/report/generate/gen_report.py +python3 /root/report/dispatch/send_report.py -- Gitee From 5c44d292bc599f77f4006e3adef1b2e982aa7b95 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 11:05:17 +0800 Subject: [PATCH 55/64] Fix: logic error on general status Signed-off-by: XZhouQD --- generate/subtask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/subtask.py b/generate/subtask.py index 6d096ab..2013ed2 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -41,7 +41,7 @@ class Subtask: fail_output = '\033[31m FAIL \033[0m' pass_output = '\033[32m PASS \033[0m' return f'\t{self.producer:16s}{self.image:42s}' \ - + f'{fail_output if self.status else pass_output}' + + f'{pass_output if self.status else fail_output}' def detail_status(self): newline = '\n' -- Gitee From fc03667f522e070e83ccd898b22ba9b190565016 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 11:58:08 +0800 Subject: [PATCH 56/64] Fix: error log logic error Signed-off-by: XZhouQD --- generate/subtask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generate/subtask.py b/generate/subtask.py index 2013ed2..3ce5fd4 100644 --- a/generate/subtask.py +++ b/generate/subtask.py @@ -51,4 +51,4 @@ class Subtask: Log: {self.log} Time: {self.start_time} ~ {self.end_time} Cost: {self.duration} -{newline.join(('Tail of log:', self.error)) if self.status else ''}""" +{newline.join(('Tail of log:', self.error)) if not self.status else ''}""" -- Gitee From 489035484897c332ec78e3a5269032757aaade2e Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 14:13:04 +0800 Subject: [PATCH 57/64] Enhancement: Add general result count for report Signed-off-by: XZhouQD --- generate/gen_report.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/generate/gen_report.py b/generate/gen_report.py index 7b09a68..1045c10 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -157,12 +157,20 @@ def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: solution_tasks = organise_tasks(main_tasks) solution_reports = {} for solution, sol_main_tasks in solution_tasks.items(): + ps = 0 + fl = 0 report_infos_brief = [] report_infos_detail = [] for task in sol_main_tasks: + if task.fail > 0: + fl+= 1 + else: + ps += 1 report_infos_brief.append(str(task) + task.subtasks_info_general()) report_infos_detail.append(task.subtasks_info_detail()) - solution_reports[solution] = '\n'.join(sorted(report_infos_brief)) \ + solution_reports[solution] = f'\033[1mTotal {ps} PASS {fl} FAIL\033[0m' \ + + f'\n{"-" * 40}\n' \ + + '\n'.join(sorted(report_infos_brief)) \ + f'\n\n{"=" * 160}\n\n\n' \ + '\t\t\033[1m[Subtask Detail Report]\033[0m\n\n\n' \ + '\n'.join(sorted(report_infos_detail)) @@ -196,7 +204,7 @@ def write_reports( for solution, report in reports.items(): file_path = f'{report_path}/{solution}_daily_report.html' with open(file_path, 'w') as fp: - content = f'\t\t\033[1m[{solution} {local_date}' \ + content = f'\t\t\033[1m[{solution} {local_date} ' \ + 'Build Brief Report]\033[0m\n' \ + report content_html = converter.convert(content) -- Gitee From a88c800d02d0880bcbdebdf79485f984f821365f Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 14:16:28 +0800 Subject: [PATCH 58/64] Fix: typo and formatting Signed-off-by: XZhouQD --- README.md | 6 +++--- generate/gen_report.py | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 9ae3716..ac4cd98 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ "SolutionName1": { "receiver":[ "receiver1@example.com", - "receiver2@example.com + "receiver2@example.com" ], "cc_receiver":[ "cc1@example.com", @@ -36,7 +36,7 @@ "SolutionName2": { "receiver":[ "receiver1@example.com", - "receiver2@example.com + "receiver2@example.com" ], "cc_receiver":[ "cc1@example.com", @@ -46,7 +46,7 @@ "SolutionName3": { "receiver":[ "receiver1@example.com", - "receiver2@example.com + "receiver2@example.com" ], "cc_receiver":[ "cc1@example.com", diff --git a/generate/gen_report.py b/generate/gen_report.py index 1045c10..e484236 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -112,7 +112,7 @@ def read_subtask_log(subtask_logs: List[str], subtask_output = line.split()[-1] if "Build subtask result:" in line: subtask_result = True if line.split( - )[-1] == 'PASS' else False + )[-1] == 'PASS' else False if 'Build subtask start time' in line: subtask_start = ' '.join(line.split()[-2:]).strip() if 'Build subtask end time' in line: @@ -163,12 +163,13 @@ def generate_report(main_tasks: List[Maintask]) -> Dict[str, str]: report_infos_detail = [] for task in sol_main_tasks: if task.fail > 0: - fl+= 1 + fl += 1 else: ps += 1 report_infos_brief.append(str(task) + task.subtasks_info_general()) report_infos_detail.append(task.subtasks_info_detail()) - solution_reports[solution] = f'\033[1mTotal {ps} PASS {fl} FAIL\033[0m' \ + solution_reports[solution] = '\n\n' \ + + f'\033[1mTotal {ps} PASS {fl} FAIL\033[0m' \ + f'\n{"-" * 40}\n' \ + '\n'.join(sorted(report_infos_brief)) \ + f'\n\n{"=" * 160}\n\n\n' \ -- Gitee From 934735a8960f21b5f9e213a8cfe4d8483fa5e67c Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 14:24:50 +0800 Subject: [PATCH 59/64] Enhancement: Add total result in report and email subject Signed-off-by: XZhouQD --- dispatch/send_report.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/dispatch/send_report.py b/dispatch/send_report.py index 8e1380d..97173ac 100644 --- a/dispatch/send_report.py +++ b/dispatch/send_report.py @@ -1,5 +1,6 @@ #!/usr/bin/python3 import json +import re import sys import time from typing import Dict @@ -23,15 +24,25 @@ def read_solution_email_receiver() -> Dict[str, str]: def dispatch_email(date, solution_receivers, smtp_client): base_dir = f'/result/autotest/reports/{date}' + ps_fl_pattern = re.compile(r'TOTAL ([0-9]+) PASS ([0-9]+) FAIL') for solution, receivers in solution_receivers.items(): file_path = f'{base_dir}/{solution}_daily_report.html' + with open(file_path, 'r') as fp: + lines = fp.readlines() + subject = f'[{date}] {solution} dockerfile build report' + for line in lines: + ps_fl_match = ps_fl_pattern.search(line) + if ps_fl_match: + ps = ps_fl_match.group(1) + fl = ps_fl_match.group(2) + subject += f' ({ps} PASS {fl} FAIL)' receiver = receivers.get('receiver') cc_receiver = receivers.get('cc_receiver') smtp_client.send_email( receiver, cc_receiver, file_path, - f'[{date}] {solution} dockerfile build report') + subject) if __name__ == '__main__': -- Gitee From f8442553c949693afe3057bbbad97808fa6c8317 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 14:27:19 +0800 Subject: [PATCH 60/64] Fix: typo in regex Signed-off-by: XZhouQD --- dispatch/send_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dispatch/send_report.py b/dispatch/send_report.py index 97173ac..30e0c34 100644 --- a/dispatch/send_report.py +++ b/dispatch/send_report.py @@ -24,7 +24,7 @@ def read_solution_email_receiver() -> Dict[str, str]: def dispatch_email(date, solution_receivers, smtp_client): base_dir = f'/result/autotest/reports/{date}' - ps_fl_pattern = re.compile(r'TOTAL ([0-9]+) PASS ([0-9]+) FAIL') + ps_fl_pattern = re.compile(r'Total ([0-9]+) PASS ([0-9]+) FAIL') for solution, receivers in solution_receivers.items(): file_path = f'{base_dir}/{solution}_daily_report.html' with open(file_path, 'r') as fp: -- Gitee From 2285c76e04d46f0c171717c959e20cf312ee77da Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 16:05:17 +0800 Subject: [PATCH 61/64] Update: README.md, add detailed service usage Signed-off-by: XZhouQD --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ac4cd98..6a8622f 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,7 @@ #### 日志发送配置方法 1. 克隆此仓库到本地（如已克隆，可忽略） -2. 修改reportsend.service与report_generate_dispatch.sh，使绝对路径正确匹配。修改reportsend.timer设置定时发送时间 -3. 复制reportsend.service与reportsend.timer到/usr/lib/systemd/system/内，启动/设置自启动此timer -4. 为解决方案、收件人创建配置文件 dispatch/config.json，如下： +2. 为解决方案、收件人创建配置文件 dispatch/config.json，如下： ```json { "SolutionName1": { @@ -58,6 +56,14 @@ #### 使用教程 +手动执行： + 1. 执行 generate/gen_report.py 以生成报告等 2. 执行 dispatch/send_report.py 以发送邮件通知 +自动定时执行： + +1. 修改reportsend.service与report_generate_dispatch.sh，使绝对路径正确匹配，并修改reportsend.timer设置定时发送时间 +2. 复制reportsend.service与reportsend.timer到/usr/lib/systemd/system/内 +3. 使用systemctl启用reportsend.service与reportsend.timer + -- Gitee From 7b991f10cbafef29f64dc0ac2729e32d9633a69e Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 7 Dec 2020 16:16:36 +0800 Subject: [PATCH 62/64] Fixed: Add report detection to avoid empty content sent Signed-off-by: XZhouQD --- README.md | 6 +++--- dispatch/send_report.py | 4 ++++ generate/gen_report.py | 3 +-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 6a8622f..5a72aa8 100644 --- a/README.md +++ b/README.md @@ -58,12 +58,12 @@ 手动执行： -1. 执行 generate/gen_report.py 以生成报告等 -2. 执行 dispatch/send_report.py 以发送邮件通知 +1. 执行 generate/gen_report.py [date] 以生成报告等。日期格式为YYYYMMDD，如20201207。 +2. 执行 dispatch/send_report.py [date] 以发送邮件通知。日期格式为YYYYMMDD，如20201207。 自动定时执行： 1. 修改reportsend.service与report_generate_dispatch.sh，使绝对路径正确匹配，并修改reportsend.timer设置定时发送时间 2. 复制reportsend.service与reportsend.timer到/usr/lib/systemd/system/内 -3. 使用systemctl启用reportsend.service与reportsend.timer +3. 使用systemctl启用reportsend.service与reportsend.timer，将在定时时间自动生成、发送本日报告 diff --git a/dispatch/send_report.py b/dispatch/send_report.py index 30e0c34..5144d8c 100644 --- a/dispatch/send_report.py +++ b/dispatch/send_report.py @@ -1,6 +1,7 @@ #!/usr/bin/python3 import json import re +import os import sys import time from typing import Dict @@ -27,6 +28,8 @@ def dispatch_email(date, solution_receivers, smtp_client): ps_fl_pattern = re.compile(r'Total ([0-9]+) PASS ([0-9]+) FAIL') for solution, receivers in solution_receivers.items(): file_path = f'{base_dir}/{solution}_daily_report.html' + if not os.path.exists(file_path): + continue with open(file_path, 'r') as fp: lines = fp.readlines() subject = f'[{date}] {solution} dockerfile build report' @@ -36,6 +39,7 @@ def dispatch_email(date, solution_receivers, smtp_client): ps = ps_fl_match.group(1) fl = ps_fl_match.group(2) subject += f' ({ps} PASS {fl} FAIL)' + break receiver = receivers.get('receiver') cc_receiver = receivers.get('cc_receiver') smtp_client.send_email( diff --git a/generate/gen_report.py b/generate/gen_report.py index e484236..b8ae8fd 100644 --- a/generate/gen_report.py +++ b/generate/gen_report.py @@ -233,8 +233,7 @@ if __name__ == '__main__': else: has_build, local_date, log_path, output_path = begin() if not has_build: - # exit no. 2: no build today - sys.exit(2) + sys.exit(0) # find available log files main_logs, subtask_logs = find_log_files(local_date, log_path) -- Gitee From 80f0403259ece66631d3cc08af697e065b78164e Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Wed, 9 Dec 2020 14:50:05 +0800 Subject: [PATCH 63/64] Fix: decrease report send frequency, add smtp error output if fail Signed-off-by: XZhouQD --- dispatch/smtp.py | 1 + reportsend.timer | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/dispatch/smtp.py b/dispatch/smtp.py index 96ac6eb..e25f2e6 100644 --- a/dispatch/smtp.py +++ b/dispatch/smtp.py @@ -56,6 +56,7 @@ class SMTP: return True except smtplib.SMTPException as e: # retry failed, give up + print(f'Send {message["Subject"]} failed due to {e}') pass return False diff --git a/reportsend.timer b/reportsend.timer index 69f58c6..0777ff9 100644 --- a/reportsend.timer +++ b/reportsend.timer @@ -1,9 +1,9 @@ [Unit] -Description=Runs reportsend every day +Description=Runs reportsend periodically [Timer] -OnCalendar=*-*-* 11:50:00 +OnCalendar=Mon,Fri 11:50:00 Unit=reportsend.service [Install] -WantedBy=multi-user.target \ No newline at end of file +WantedBy=multi-user.target -- Gitee From 5fd6617a49c89991fb911efe9c0c830acb572fe3 Mon Sep 17 00:00:00 2001 From: XZhouQD Date: Mon, 14 Dec 2020 14:57:38 +0800 Subject: [PATCH 64/64] Fix: Try update format of receiver and cc sender to avoid DT:SPM Signed-off-by: XZhouQD --- dispatch/smtp.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dispatch/smtp.py b/dispatch/smtp.py index e25f2e6..90aa3ab 100644 --- a/dispatch/smtp.py +++ b/dispatch/smtp.py @@ -35,8 +35,10 @@ class SMTP: content = fp.read() message = MIMEMultipart() message['From'] = self.sender + receiver = ['<' + email_addr + '>' for email_addr in receiver] message['To'] = ','.join(receiver) - message['CC'] = ','.join(cc_receiver) + cc_receiver = ['<' + email_addr + '>' for email_addr in cc_receiver] + message['CC'] = ','.join(cc_receiver + [f'<{self.sender}>']) message['Subject'] = subject message.attach(MIMEText(content, 'html', 'utf-8')) attachment = MIMEText(content, 'base64', 'utf-8') -- Gitee