From 464eed498cfe6c1fe8746ae8fb6eacfefba065d4 Mon Sep 17 00:00:00 2001 From: Ronnie_Jiang Date: Sun, 14 Mar 2021 21:45:35 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8F=90=E4=BA=A4issue=5Freport.py=E6=96=87?= =?UTF-8?q?=E4=BB=B6=EF=BC=8C=E6=94=AF=E6=8C=81=E8=87=AA=E5=8A=A8=E7=94=9F?= =?UTF-8?q?=E6=88=90issue=E7=AE=A1=E7=90=86=E6=8A=A5=E5=91=8A=E5=8A=9F?= =?UTF-8?q?=E8=83=BD=EF=BC=8C=E5=90=8C=E6=97=B6=E5=88=B7=E6=96=B0README.md?= =?UTF-8?q?=E6=96=87=E6=A1=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 16 ++ advisors/issue_report.py | 383 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 399 insertions(+) create mode 100644 advisors/issue_report.py diff --git a/README.md b/README.md index f9c25f70..7c502d7b 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,10 @@ openEuler-Advisor 的目标是为 openEuler 制品仓的日常工作提供自动 2.10、review_tool.py 用于生成指定软件仓PR的代码审视清单，以规范化PR审视过程。 +2.11、issue_report.py 是一款自动生成版本issue及cve管理表格(csv)和结果报告(markdown)的工具，提供基于openeuler版本生成issue和cve清单的功能。 + 3、prow：存放对接CI/CD框架PROW的脚本 + ## 2、后续计划 1、@solarhu 团队正在开发工具，计划提供 openEuler 内所有组件依赖关系的查询。 @@ -208,5 +211,18 @@ tag中版本的间隔符，如果 tag是 v1_0_1，然后配置separator 为"_" 查询软件包上游社区信息及版本推荐: python3 oa_upgradable.py pkg_name 例如: python3 oa_upgradable.py glibc +##### c. issue_report.py + 运行环境：python版本 3.8及以上 + 工具使用： + + ```bash + python3 issue_report.py -milestone "openEuler 20.03-LTS" "openEuler 20.09" -branch "openEuler-21.03" "openEuler-20.09" -outpath /Users/lilu/Downloads + ``` + + > 参数说明： + > -milestone: openEuler版本的里程碑，支持输入多个里程碑名。例如："openEuler-21.03", "openEuler 21.03-RC1"等 + > -branch: src-openEuler仓库下的具体分支名称，支持输入多个分支名。例如："openEuler-21.03", "openEuler-20.09"等 + > -outpath: 版本管理报告和版本发布报告生成路径 + #### 3.2.3 advisors咨询: 如果有其他问题或疑问，请邮件联系：licihua@huawei.com/zwfeng@huawei.com/shanshishi@huawei.com diff --git a/advisors/issue_report.py b/advisors/issue_report.py new file mode 100644 index 00000000..f40800ae --- /dev/null +++ b/advisors/issue_report.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 + +""" +Description:generate issue report +""" +# pylint: disable=inconsistent-return-statements +# pylint: disable=broad-except +# pylint: disable=no-else-return +# pylint: disable=too-many-locals +import copy +import os +import sys +import json +import pandas as pd +import requests + +FILE_NAME = sys.argv[0] +CMD_INPUT_ARGS = sys.argv[1:] + + +def _to_markdown(df: pd.DataFrame, index=False): + cols = df.columns + title = "|" + "|".join(str(col) for col in cols) + "|\n" + under_title = "|" + "---|" * len(cols) + "\n" + + content = "" + for idx, row in df.iterrows(): + curr_row = list(str(r) for r in row) + if index: + curr_row.insert(idx) + content += "|" + "|".join(curr_row) + "|\n" + + return title + under_title + content + +def parse_args(name, input_args): + """ + 解析输入的参数 + Args: + name: 运行文件名 + input_args: 解析出来的参数 + + Returns: + issue_params, cve_params, output_path + """ + split_index = 0 + if "-milestone" not in input_args or "-branch" not in input_args: + print( + "usage: {} [-milestone [Required parameters]] [-branch [Required parameters]] " + "[-outpath [Optional parameters]]".format(name) + ) + return + if "-outpath" not in input_args and ( + "/" in input_args[-1] or "\\" in input_args[-1] + ): + print( + "Missing -outpath \n" + "usage: {} [-milestone [Required parameters]] [-branch [Required parameters]] " + "[-outpath [Optional parameters]]".format(name) + ) + return + for index, con in enumerate(input_args): + if con == "-branch": + split_index = index + issue_params = input_args[1:split_index] + cve_params = ( + input_args[split_index + 1 : len(input_args) - 2] + if "-outpath" in input_args + else input_args[split_index + 1 :] + ) + output_path = input_args[-1] if "-outpath" in input_args else "./" + if not os.path.exists(output_path): + print("The output path is illegal, please re-enter!") + return + return issue_params, cve_params, output_path + + +def build_request_parameters(issue_params, cve_params): + """ + 构建请求参数 + Args: + issue_params: Issue请求参数 + cve_params: cve请求参数 + + Returns: + issue_params_list, cve_params_list + """ + parameters_issue = { + "community": "openeuler", + "state": "all", + "milestone": "", + "sortKey": "closed_at", + "sortValue": "descending", + } + parameters_cve = { + "community": "openeuler", + "state": "all", + "milestone": "", + "sortKey": "closed_at", + "sortValue": "descending", + } + issue_params_list, cve_params_list = [], [] + for issue_con in issue_params: + parameters_issue["milestone"] = issue_con + issue_params_list.append(copy.copy(parameters_issue)) + for cve_con in cve_params: + parameters_cve["milestone"] = cve_con + cve_params_list.append(copy.copy(parameters_cve)) + return issue_params_list, cve_params_list + + +def query_issue_response(parameters_issue): + """ + 获取issue响应 + Args: + parameters_issue: 请求参数 + + Returns: + response_issue_string + """ + url_issue = "http://omapi.osinfra.cn/IssueData" + try: + response_issue = requests.post( + url_issue, + data=json.dumps(parameters_issue), + headers={"Content-type": "application/json"}, + ) + except Exception as error: + print(error) + return None + if response_issue.status_code == 200: + response_issue_string = response_issue.json()["data"] + return response_issue_string + else: + print(response_issue.status_code) + return [] + + +def query_cve_response(parameters_cve): + """ + 获取cve响应 + Args: + parameters_cve: 请求参数 + + Returns: + response_cve_string + """ + url_cve = "http://omapi.osinfra.cn/CVEData" + try: + response_cve = requests.post( + url_cve, + data=json.dumps(parameters_cve), + headers={"Content-type": "application/json"}, + ) + except Exception as error: + print(error) + return None + if response_cve.status_code == 200: + response_cve_string = response_cve.json()["data"] + return response_cve_string + else: + print(response_cve.status_code) + return [] + + +# pylint: disable=unused-variable +def filter_cve_content(sources): + """ + 去除cve数据中type字段不是“CVE和安全问题”的元素 + Args: + sources: cve响应数据 + + Returns: + filter_source + """ + filter_source = [] + for index, response in enumerate(sources): + if response.get("type") == "CVE和安全问题": + filter_source.append(response) + return filter_source + + +def generate_csv(issue_sources, cve_sources, output_path): + """ + 将获取的issue数据按照模板输出csv文件 + Args: + issue_sources: issue响应数据 + cve_sources: cve响应数据 + output_path: 输出路径 + + Returns: + csv文件 + """ + issue_id = [] + issue_type = [] + issue_title = [] + owner = [] + state = [] + plan_deadline_at = [] + closed_at = [] + progress = [] + version = [] + + for response in issue_sources: + issue_id.append(response.get("issue_id", "暂无相应信息")) + issue_type.append(response.get("type", "暂无相应信息")) + issue_title.append(response.get("issue_title", "暂无相应信息")) + owner.append(response.get("assignee_name", "暂无相应信息")) + state.append(response.get("state", "暂无相应信息")) + plan_deadline_at.append(response.get("plan_deadline_at", "暂无相应信息")) + closed_at.append(response.get("closed_at", "暂无相应信息")) + progress.append(response.get("plan_start_at", "暂无相应信息")) + version.append(response.get("milestone", "暂无相应信息")) + for response in cve_sources: + issue_id.append(response.get("issue_id", "暂无相应信息")) + issue_type.append(response.get("type")) + issue_title.append(response.get("issue_title", "暂无相应信息")) + owner.append(response.get("assignee_name", "暂无相应信息")) + state.append(response.get("state", "暂无相应信息")) + progress.append(response.get("plan_start_at", "暂无相应信息")) + plan_deadline_at.append(response.get("plan_deadline_at", "暂无相应信息")) + closed_at.append(response.get("closed_at", "暂无相应信息")) + version.append( + ",".join( + [ + con.replace(":受影响", "").replace(":不受影响", "") + for con in response["milestone"].split(",") + ] + ) + ) + # 字典中的key值即为csv中列名 + dataframe = pd.DataFrame( + { + "Issue": issue_id, + "类型": issue_type, + "概述": issue_title, + "责任人": owner, + "状态": state, + "计划合入时间": plan_deadline_at, + "实际关闭时间": closed_at, + "进展": progress, + "所属版本": version, + } + ) + dataframe.to_csv(output_path + "/Issue管理报告.csv", index=False, encoding="utf_8_sig") + + +def generate_md(issue_sources, cve_sources, output_path): + """ + 将获取的issue数据按模板输出MD文件 + Args: + issue_sources: issue响应数据 + cve_sources: cve响应数据 + output_path: 输出路径 + + Returns: + MD文件 + """ + description_feature_list = "## 特性清单" + description_issue_list = "## 解决问题清单" + description_todo_list = "## 遗留问题清单" + description_cve_list = "## 解决CVE清单" + feature_id = [] + feature_issue_title = [] + feature_version = [] + fixed_issue_id = [] + fixed_issue_title = [] + fixed_version = [] + todo_issue_id = [] + todo_issue_title = [] + effect = [] + todo_version = [] + for response in issue_sources: + if response.get("type") == "需求": + feature_id.append(response.get("issue_id", "暂无相应信息")) + feature_issue_title.append(response.get("issue_title", "暂无相应信息")) + feature_version.append(response.get("milestone", "暂无相应信息")) + elif response.get("type") == "缺陷" and response.get("state") == "closed": + fixed_issue_id.append(response.get("issue_id", "暂无相应信息")) + fixed_issue_title.append(response.get("issue_title", "暂无相应信息")) + fixed_version.append(response.get("milestone", "暂无相应信息")) + elif response.get("type") == "缺陷" and response.get("state") == "open": + todo_issue_id.append(response.get("issue_id", "暂无相应信息")) + todo_issue_title.append(response.get("issue_title", "暂无相应信息")) + effect.append(response.get("description", "暂无相应信息")) + todo_version.append(response.get("milestone", "暂无相应信息")) + + dataframe_feature = pd.DataFrame( + {"Issue": feature_id, "概述": feature_issue_title, "所属版本": feature_version} + ) + dataframe_feature_str = _to_markdown(dataframe_feature) + + dataframe_fixed_issue = pd.DataFrame( + {"Issue": fixed_issue_id, "概述": fixed_issue_title, "所属版本": fixed_version} + ) + + count = 0 + for col in dataframe_fixed_issue: + if count < 5: + print(col) + + count += 1 + dataframe_fixed_issue_str = _to_markdown(dataframe_fixed_issue) + + dataframe_todo_issue = pd.DataFrame( + { + "Issue": todo_issue_id, + "概述": todo_issue_title, + "影响分析": effect, + "所属版本": todo_version, + } + ) + + + dataframe_todo_issue_str = _to_markdown(dataframe_todo_issue) + + cve_id = [] + openeuler_score = [] + cve_issue_title = [] + cve_version = [] + for response in cve_sources: + cve_id.append(response.get("issue_id", "暂无相应信息")) + openeuler_score.append(response.get("openeuler_score", "暂无相应信息")) + cve_issue_title.append(response.get("issue_title", "暂无相应信息")) + cve_version.append( + ",".join( + [ + con.replace(":受影响", "").replace(":不受影响", "") + for con in response["milestone"].split(",") + ] + ) + ) + dataframe_cve = pd.DataFrame( + { + "Issue": cve_id, + "分值": openeuler_score, + "概述": cve_issue_title, + "所属版本": cve_version, + } + ) + dataframe_cve_str = _to_markdown(dataframe_cve) + + string = ( + description_feature_list, + dataframe_feature_str, + description_issue_list, + dataframe_fixed_issue_str, + description_todo_list, + dataframe_todo_issue_str, + description_cve_list, + dataframe_cve_str, + ) + strings = "\n".join(string) + + with open(output_path + "/Issue结果报告.md", "w") as text_file: + text_file.write(strings) + + +# pylint: disable=broad-except +def main(): + """ + Function main program + Returns: + + """ + try: + issue_params, cve_params, output_path = parse_args(FILE_NAME, CMD_INPUT_ARGS) + except Exception as error: + print(error) + else: + parameters_issues, parameters_cves = build_request_parameters( + issue_params, cve_params + ) + issue_con, cve_con = [], [] + for parameters_issue in parameters_issues: + issue_con.extend(query_issue_response(parameters_issue)) + for parameters_cve in parameters_cves: + cve_con.extend(query_cve_response(parameters_cve)) + filter_cve_con = filter_cve_content(cve_con) + generate_csv(issue_con, filter_cve_con, output_path) + generate_md(issue_con, filter_cve_con, output_path) + + +main() -- Gitee