diff --git a/conf/app.conf b/conf/app.conf index 755aae339592245acadab2ce73bccf4f0a92be18..b25442fdfd66343061623545b7b340ccc72a3f8e 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -46,13 +46,13 @@ maxsize=204800 [crontab] ymalflag = 2 getymal = 00 00 23 * * * -cveflag = 1 +cveflag = 2 getcve = 00 00 01 * * * oricveflag = 2 oricvecheck = 00 00 05 * * * getissueflag = 2 getissue = 00 00 05 * * * -issueflag = 1 +issueflag = 2 createissue = * * 05 * * * test = 0/10 * * * * * gittokenflag = 2 diff --git a/cve-py/controller/taskcontroller.py b/cve-py/controller/taskcontroller.py index 0431de82af145aa86ada62a0ffa5d22f16f2aee9..d0d5d269efc9c2e16750b1a30a404b7ee872e43a 100644 --- a/cve-py/controller/taskcontroller.py +++ b/cve-py/controller/taskcontroller.py @@ -13,7 +13,7 @@ Authors: xiaojianghui Date: 10/22/2020 11:01 AM """ -from tabletask import runtask, mappingtask, toexcel, export_excel_task, import_excel_task +from tabletask import runtask, mappingtask, toexcel, export_excel_task, import_excel_task, package_committer_task from gitwebtask import genegroup, yamltask from emailtask import sendingtask, issue_record_email from deletetask import deletefiletask @@ -107,3 +107,14 @@ def run_whiltlist_task(): print('the task of parsing package whitelist start') import_excel_task.import_data() print('the task of parsing package whitelist complete') + + +def run_package_committer_task(): + """ + the task of parsing package_committer excels + Returns:None + + """ + print('the task of parsing package_committer_excels start') + package_committer_task.import_data() + print('the task of parsing package_committer_excels complete') diff --git a/cve-py/controller/timertaskcontroller.py b/cve-py/controller/timertaskcontroller.py index e18c33f0b4877f16f18a252c5f9d378a2a1b45f4..a72572126d048d0ce972e4d0717a30b2b37e12ef 100644 --- a/cve-py/controller/timertaskcontroller.py +++ b/cve-py/controller/timertaskcontroller.py @@ -39,6 +39,8 @@ def timertask(): scheduler.add_job(taskcontroller.run_issue_record_task, 'cron', day_of_week='0-6', hour=7, minute=30) # analysis the table package whitelist scheduler.add_job(taskcontroller.run_whiltlist_task, 'interval', hours=2) + # parsing the table package_committer_excels + scheduler.add_job(taskcontroller.run_package_committer_task, 'interval', hours=1) scheduler.start() except SystemExit as err: print("Err:", err) diff --git a/cve-py/downloadtask/downloadfiletask.py b/cve-py/downloadtask/downloadfiletask.py index b2667a3d6a49af0f808fef1e1ba22df09e68be89..c91c82de7d9e807265ccb9d43114febfe54af162 100644 --- a/cve-py/downloadtask/downloadfiletask.py +++ b/cve-py/downloadtask/downloadfiletask.py @@ -123,3 +123,38 @@ def handle_three(): if os.path.exists("./import_excels/" + name): print("Download the Package whitelist table successfully:" + name) i += 2 + + +def handle_four(): + """ + download file + """ + if os.listdir("./package_committer_excels"): + shutil.rmtree("./package_committer_excels") + os.mkdir("./package_committer_excels") + url = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/package_committer_excels' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/package_committer_excels/" + name + r = requests.get(url) + print(r.status_code) + with open("./package_committer_excels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./package_committer_excels/" + name): + print("Download the package_committer table successfully:" + name) + i += 2 diff --git a/cve-py/import_excels/21_whitelist_2020-11-21.xlsx b/cve-py/import_excels/21_whitelist_2020-11-21.xlsx deleted file mode 100644 index 98b98e3761efd81dc098112c10f1450dfc6cd76c..0000000000000000000000000000000000000000 Binary files a/cve-py/import_excels/21_whitelist_2020-11-21.xlsx and /dev/null differ diff --git "a/cve-py/package_committer_excels/openEuler\347\244\276\345\214\272\350\275\257\344\273\266\345\214\205\350\264\243\344\273\273\347\224\260-openeuler.xlsx" "b/cve-py/package_committer_excels/openEuler\347\244\276\345\214\272\350\275\257\344\273\266\345\214\205\350\264\243\344\273\273\347\224\260-openeuler.xlsx" new file mode 100644 index 0000000000000000000000000000000000000000..efdab9447513e025e5efa1d38ef821f9ca36b3b3 Binary files /dev/null and "b/cve-py/package_committer_excels/openEuler\347\244\276\345\214\272\350\275\257\344\273\266\345\214\205\350\264\243\344\273\273\347\224\260-openeuler.xlsx" differ diff --git a/cve-py/tabletask/package_committer_task.py b/cve-py/tabletask/package_committer_task.py new file mode 100644 index 0000000000000000000000000000000000000000..48b240100611906161dc92617dc0237bc1b45e1c --- /dev/null +++ b/cve-py/tabletask/package_committer_task.py @@ -0,0 +1,92 @@ +#!/usr/bin/python3 +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +parse excel file and mysql data + +Authors:xiaojianghui +Date: 11/27/2020 11:01 AM +""" + +import os +import xlrd +import time +import hashlib +from dbConnecttion.MysqlConn import Mysql +from downloadtask import downloadfiletask + + +def parse_excel(file_name): + """ + parse excel data + parse: file_name + return: list + """ + package_name_list = [] + issue_assignee_list = [] + try: + data = xlrd.open_workbook('./package_committer_excels/' + file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + package_name_list.append(table_one.cell(i, 0).value) + issue_assignee_list.append(table_one.cell(i, 1).value) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return package_name_list, issue_assignee_list + + +def cur_date(): + """ + current date + :return createTime: string + """ + create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + return create_time + + +def import_data(): + """ + import excel + :param + :return none + """ + downloadfiletask.handle_four() + mysql = Mysql() + files = os.listdir('./package_committer_excels') + for filename in files: + with open('./package_committer_excels/' + filename, 'rb') as f: + sha1obj = hashlib.sha1() + sha1obj.update(f.read()) + hash_value = sha1obj.hexdigest() + print(filename, hash_value) + f.close() + sql = "select file_hash from cve_file_hash where file_name = %s" + val = (filename,) + file_hash = mysql.getOne(sql, val) + if file_hash: + if hash_value == file_hash['file_hash']: + print("文件已解析:" + filename) + os.remove('./package_committer_excels/' + filename) + continue + sql = "insert into cve_file_hash (file_name, file_hash) values (%s, %s)" + val = (filename, hash_value) + mysql.insertOne(sql, val) + mysql.dispose() + result = parse_excel(filename) + for i in range(0, len(result[0])): + sql = 'insert ignore into cve_spec_issue_assigness (package_name, issue_assignee, status, create_time) ' \ + 'values (%s, %s, %s, %s)' + val = (result[0][i], result[1][i], 1, cur_date()) + mysql.insertOne(sql, val) + mysql.dispose() + print('插入数据{}:{}'.format(result[0][i], result[1][i])) + os.remove('./package_committer_excels/' + filename) + mysql.close() diff --git a/doc/sql/db_struct.sql b/doc/sql/db_struct.sql index 420bfe82b5d79ea2d57bfd4e4a23c8a0056d63a5..e3b03c85221c151e4a91ae1b58709c63c835ba33 100644 --- a/doc/sql/db_struct.sql +++ b/doc/sql/db_struct.sql @@ -52,7 +52,18 @@ CREATE TABLE `cve_export_record` ( `state` tinyint NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `file_name` (`file_name`) -) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=55 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_file_hash` */ + +DROP TABLE IF EXISTS `cve_file_hash`; + +CREATE TABLE `cve_file_hash` ( + `id` int NOT NULL AUTO_INCREMENT, + `file_name` varchar(50) DEFAULT NULL, + `file_hash` varchar(50) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_open_euler` */ @@ -86,7 +97,7 @@ CREATE TABLE `cve_git_open_euler` ( KEY `cve_git_open_euler_package_id` (`package_id`), KEY `cve_git_open_euler_version` (`version`), KEY `cve_git_open_euler_table_id` (`table_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4748 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=5071 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_open_euler_table_relate` */ @@ -241,7 +252,7 @@ CREATE TABLE `cve_gite_origin_issue` ( PRIMARY KEY (`id`), UNIQUE KEY `issue_id` (`issue_id`), UNIQUE KEY `number` (`number`) -) ENGINE=InnoDB AUTO_INCREMENT=591 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=4190 DEFAULT CHARSET=utf8; /*Table structure for table `cve_gite_repo` */ @@ -262,7 +273,7 @@ CREATE TABLE `cve_gite_repo` ( `delete_time` varchar(255) DEFAULT NULL, PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_group_id` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=28649 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=28651 DEFAULT CHARSET=utf8; /*Table structure for table `cve_gite_repo_branch` */ @@ -293,7 +304,7 @@ CREATE TABLE `cve_gite_repo_member` ( PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_member_group_id` (`group_id`), CONSTRAINT `member_groups_group_id` FOREIGN KEY (`group_id`) REFERENCES `cve_git_repo_groups` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1766 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=1768 DEFAULT CHARSET=utf8; /*Table structure for table `cve_ip_white` */ @@ -317,7 +328,31 @@ CREATE TABLE `cve_issue_assignee` ( `assignee` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY (`id`), UNIQUE KEY `assignee` (`assignee`) -) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_create_record` */ + +DROP TABLE IF EXISTS `cve_issue_create_record`; + +CREATE TABLE `cve_issue_create_record` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `cve_num` varchar(256) NOT NULL DEFAULT '', + `cve_desc` varchar(8192) NOT NULL DEFAULT '', + `cve_level` varchar(32) NOT NULL DEFAULT '', + `cve_version` varchar(128) NOT NULL DEFAULT '', + `repair_time` varchar(32) NOT NULL DEFAULT '', + `pack_name` varchar(512) NOT NULL DEFAULT '', + `nvd_score` decimal(10,1) NOT NULL DEFAULT '0.0', + `n_vector_value` varchar(256) NOT NULL DEFAULT '', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + `status` tinyint NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + KEY `cve_issue_create_record_cve_id` (`cve_id`), + KEY `cve_issue_create_record_cve_num` (`cve_num`) +) ENGINE=InnoDB AUTO_INCREMENT=1425 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_hooks` */ @@ -345,7 +380,25 @@ CREATE TABLE `cve_issue_hooks` ( UNIQUE KEY `cve_issue_hooks_owner_repo_status_un` (`owner`,`repo`,`hook_url`,`status`), KEY `cve_issue_hooks_Cve_id` (`cve_id`), KEY `cve_issue_hooks_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=21 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=27 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_repo_whitelist` */ + +DROP TABLE IF EXISTS `cve_issue_repo_whitelist`; + +CREATE TABLE `cve_issue_repo_whitelist` ( + `repo_id` bigint NOT NULL AUTO_INCREMENT, + `package_name` varchar(256) NOT NULL DEFAULT '', + `version` varchar(64) NOT NULL DEFAULT '', + `branchs` varchar(512) DEFAULT NULL, + `status` tinyint NOT NULL DEFAULT '0', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`repo_id`), + KEY `cve_issue_repo_whitelist_package_name` (`package_name`), + KEY `cve_issue_repo_whitelist_version` (`version`) +) ENGINE=InnoDB AUTO_INCREMENT=143 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_template` */ @@ -387,11 +440,15 @@ CREATE TABLE `cve_issue_template` ( `update_time` datetime NOT NULL, `delete_time` datetime NOT NULL, `error_description` text CHARACTER SET utf8 COLLATE utf8_general_ci, + `mt_audit_flag` tinyint NOT NULL DEFAULT '0', + `sa_audit_flag` tinyint NOT NULL DEFAULT '0', + `op_audit_flag` tinyint NOT NULL DEFAULT '0', PRIMARY KEY (`template_id`), UNIQUE KEY `cve_issue_template_cvn_issnu` (`cve_num`,`issue_num`), + UNIQUE KEY `cve_issue_template_num_compone_versio` (`cve_num`,`owned_component`,`owned_version`), KEY `cve_issue_template_cve_id` (`cve_id`), KEY `cve_issue_template_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=625 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2051 DEFAULT CHARSET=utf8; /*Table structure for table `cve_open_euler_s_a` */ @@ -405,7 +462,25 @@ CREATE TABLE `cve_open_euler_s_a` ( PRIMARY KEY (`openeuler_id`), UNIQUE KEY `openeuler_sa_num` (`openeuler_sa_num`), KEY `cve_open_euler_s_a_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1949 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2676 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_open_guss_yaml` */ + +DROP TABLE IF EXISTS `cve_open_guss_yaml`; + +CREATE TABLE `cve_open_guss_yaml` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `package_name` varchar(256) NOT NULL DEFAULT '', + `version` varchar(64) NOT NULL DEFAULT '', + `origin_url` varchar(512) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `package_name_version` (`package_name`,`version`), + KEY `cve_open_guss_yaml_version` (`version`) +) ENGINE=InnoDB AUTO_INCREMENT=631 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_excel` */ @@ -440,7 +515,7 @@ CREATE TABLE `cve_origin_excel` ( `delete_time` datetime DEFAULT NULL, PRIMARY KEY (`cve_id`) USING BTREE, UNIQUE KEY `cve_origin_excel_num_un` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; +) ENGINE=InnoDB AUTO_INCREMENT=235 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; /*Table structure for table `cve_origin_upstream` */ @@ -473,7 +548,7 @@ CREATE TABLE `cve_origin_upstream` ( KEY `cve_origin_upstream_cve_packname` (`cve_packname`), KEY `cve_origin_upstream_git_packname` (`git_packname`), KEY `cve_origin_upstream_version` (`version`) -) ENGINE=InnoDB AUTO_INCREMENT=1457 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=1847 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config` */ @@ -485,7 +560,7 @@ CREATE TABLE `cve_origin_upstream_config` ( `nodes` varchar(32) DEFAULT NULL, PRIMARY KEY (`conf_id`), KEY `cve_origin_upstream_config_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node` */ @@ -497,7 +572,7 @@ CREATE TABLE `cve_origin_upstream_config_node` ( `operator` varchar(256) DEFAULT NULL, PRIMARY KEY (`node_id`), KEY `cve_origin_upstream_config_node_conf_id` (`conf_id`) -) ENGINE=InnoDB AUTO_INCREMENT=8024 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=24801 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node_cpe` */ @@ -511,7 +586,7 @@ CREATE TABLE `cve_origin_upstream_config_node_cpe` ( `vulner_able` varchar(64) DEFAULT NULL, PRIMARY KEY (`cpe_id`), KEY `cve_origin_upstream_config_node_cpe_node_id` (`node_id`) -) ENGINE=InnoDB AUTO_INCREMENT=246346 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=767810 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_desc` */ @@ -524,7 +599,7 @@ CREATE TABLE `cve_origin_upstream_desc` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`desc_id`), KEY `cve_origin_upstream_desc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_event` */ @@ -539,7 +614,7 @@ CREATE TABLE `cve_origin_upstream_event` ( `description` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`event_id`), KEY `cve_origin_upstream_event_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest` */ @@ -551,7 +626,7 @@ CREATE TABLE `cve_origin_upstream_fix_suggest` ( `detail` varchar(1024) DEFAULT NULL, PRIMARY KEY (`fix_id`), KEY `cve_origin_upstream_fix_suggest_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4219 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13125 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest_ref` */ @@ -589,7 +664,7 @@ CREATE TABLE `cve_origin_upstream_impact` ( `impact` varchar(32) DEFAULT NULL, PRIMARY KEY (`impact_id`), KEY `cve_origin_upstream_impact_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score` */ @@ -605,7 +680,7 @@ CREATE TABLE `cve_origin_upstream_impact_score` ( `score_status` tinyint DEFAULT '1', PRIMARY KEY (`score_id`), KEY `cve_origin_upstream_impact_score_impact_id` (`impact_id`) -) ENGINE=InnoDB AUTO_INCREMENT=8473 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=26355 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v2` */ @@ -634,7 +709,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v2` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v2_id`), KEY `cve_origin_upstream_impact_score_v2_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v3` */ @@ -660,7 +735,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v3` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v3_id`), KEY `cve_origin_upstream_impact_score_v3_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_poc` */ @@ -677,7 +752,7 @@ CREATE TABLE `cve_origin_upstream_poc` ( `desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`poc_id`), KEY `cve_origin_upstream_poc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_reference` */ @@ -692,7 +767,7 @@ CREATE TABLE `cve_origin_upstream_reference` ( `tags` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`ref_id`), KEY `cve_origin_upstream_reference_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=83782 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=260700 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_vul_type` */ @@ -706,7 +781,7 @@ CREATE TABLE `cve_origin_upstream_vul_type` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`vul_id`), KEY `cve_origin_upstream_vul_type_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4219 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13125 DEFAULT CHARSET=utf8; /*Table structure for table `cve_other_user` */ @@ -722,7 +797,7 @@ CREATE TABLE `cve_other_user` ( `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `delete_time` datetime DEFAULT NULL, PRIMARY KEY (`user_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; /*Table structure for table `cve_package` */ @@ -735,7 +810,7 @@ CREATE TABLE `cve_package` ( `pack_url` varchar(2048) DEFAULT NULL, PRIMARY KEY (`id`), KEY `cve_package_sec_id` (`sec_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=10087 DEFAULT CHARSET=utf8; /*Table structure for table `cve_package_cpe` */ @@ -794,7 +869,7 @@ CREATE TABLE `cve_score` ( PRIMARY KEY (`id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_score_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1949 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2676 DEFAULT CHARSET=utf8; /*Table structure for table `cve_score_record` */ @@ -809,7 +884,7 @@ CREATE TABLE `cve_score_record` ( `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `cve_score_record_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1951 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3450 DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_notice` */ @@ -835,7 +910,7 @@ CREATE TABLE `cve_security_notice` ( PRIMARY KEY (`sec_id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_security_notice_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1949 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2676 DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_reviewer` */ @@ -847,7 +922,23 @@ CREATE TABLE `cve_security_reviewer` ( `status` tinyint NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `name_space` (`name_space`) -) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_spec_issue_assigness` */ + +DROP TABLE IF EXISTS `cve_spec_issue_assigness`; + +CREATE TABLE `cve_spec_issue_assigness` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `package_name` varchar(256) NOT NULL DEFAULT '', + `issue_assignee` varchar(256) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `cve_spec_issue_assigness_package_name` (`package_name`) +) ENGINE=InnoDB AUTO_INCREMENT=1195 DEFAULT CHARSET=utf8; /*Table structure for table `cve_vuln_center` */ @@ -872,7 +963,7 @@ CREATE TABLE `cve_vuln_center` ( PRIMARY KEY (`cve_id`), UNIQUE KEY `cve_vuln_center_num_pack_v_un` (`cve_num`,`cve_version`,`pack_name`), KEY `cve_vuln_center_cve_num` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=1955 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2682 DEFAULT CHARSET=utf8; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; diff --git a/models/issue.go b/models/issue.go index d0280832d2c3234cfb3dc0354b2ec08edbdbee11..a08b8409835f573a261a76f99a416565da476eb7 100644 --- a/models/issue.go +++ b/models/issue.go @@ -175,6 +175,12 @@ func GetIssueTemplateByColName(it *IssueTemplate, colName ...string) error { return err } +func GetSpecIssueAssignee(sia *SpecIssueAssigness, colName ...string) error { + o := orm.NewOrm() + err := o.Read(sia, colName...) + return err +} + func UpdateIssueTemplate(it *IssueTemplate, fields ...string) error { logs.Info("===>", it, fields) o := orm.NewOrm() diff --git a/models/modeldb.go b/models/modeldb.go index ae0412b28945b24ed06fab1b73de6ecc66ab31b0..cfb5c7f977b0af8e8d830e37bfb5ec62f0bde958 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -615,6 +615,16 @@ type OpenGussYaml struct { DeleteTime string `orm:"size(32);column(delete_time);null"` } +type SpecIssueAssigness struct { + Id int64 `orm:"pk;auto"` + PackageName string `orm:"column(package_name);size(256);unique" description:"包名称"` + Assignee string `orm:"size(256);column(issue_assignee);" description:"包对应的责任人"` + Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + func CreateDb() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -637,7 +647,7 @@ func CreateDb() bool { new(OriginUpstreamFixSuggest), new(OriginUpstreamFixSuggestRefTag), new(OriginUpstreamFixSuggestRef), new(GiteOriginIssue), new(OriginExcel), new(ExportRecord), new(GitRepoGroups), new(GiteRepo), new(GiteRepoMember), new(GiteRepoBranch), new(PackageCpe), new(EmailList), new(IssueAssignee), - new(IssueRepoWhitelist), new(IssueCreateRecord), new(OpenGussYaml), + new(IssueRepoWhitelist), new(IssueCreateRecord), new(OpenGussYaml), new(SpecIssueAssigness), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/task/issuetask.go b/task/issuetask.go index e67bd4edf6be47577a72c8222013a29d8f9e502d..bf2f95d24a349b407494476e1daf19a8818cddb0 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -11,6 +11,7 @@ import ( "os" "strconv" "strings" + "sync" ) //GetGiteeToken Get Code Cloud API TOKEN @@ -74,6 +75,221 @@ func GenIssueRecordData(icr *models.IssueCreateRecord, issueValue models.VulnCen icr.CreateTime = common.GetCurTime() } +var mutex sync.Mutex + +func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears int) error { + for ; ; { + cveData, err := models.QueryIssue(beforeTime, prcnum) + if err == nil && len(cveData) > 0 { + logs.Info(cveData) + } else { + logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) + return err + } + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) + return err + } + owner := beego.AppConfig.String("gitee::owner") + path := beego.AppConfig.String("gitee::path") + for index, issueValue := range cveData { + logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) + // Determine whether the issue has been processed + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝" { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) + continue + } + } + // Import cve as data after 2018 + cveNumList := strings.Split(issueValue.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if issueValue.DataSource == 1 { + years = toolYears + } else if issueValue.DataSource == 3 { + years = manYears + } + if cveYears <= years { + icr := models.IssueCreateRecord{} + models.UpdateIssueStatus(issueValue, 4) + logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", + years, ",否则不需要在git上提交issue, cveData: ", issueValue) + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + continue + } + GenIssueRecordData(&icr, issueValue, sc, 2) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } + continue + } + } + } + // Determine whether cve has been processed + issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) + if issueExist { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) + continue + } + // Process each piece of cve data + if issueValue.Status == 0 { + mutex.Lock() + err := ProcIssue(issueValue, accessToken, owner, path) + mutex.Unlock() + if err != nil { + logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } else { + mutex.Lock() + err := ProcUpdateIssue(issueValue, accessToken, owner, path) + mutex.Unlock() + if err != nil { + logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } + // Collect issue record data + icr := models.IssueCreateRecord{} + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + continue + } + GenIssueRecordData(&icr, issueValue, sc, 1) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } + } + } +} + +func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears int) error { + issueWhiteData, issueErr := models.QueryIssueWhitelist(1) + if issueErr == nil && len(issueWhiteData) > 0 { + for _, issues := range issueWhiteData { + for ; ; { + cveData, err := models.QueryIssueByPackName(issues.PackageName, beforeTime, prcnum) + if err == nil && len(cveData) > 0 { + logs.Info(cveData) + } else { + logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) + break + } + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime(), ",err: ", err) + break + } + owner := beego.AppConfig.String("gitee::owner") + path := beego.AppConfig.String("gitee::path") + for index, issueValue := range cveData { + logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) + // Determine whether the issue has been processed + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝" { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) + continue + } + } + // Import cve as data after 2018 + cveNumList := strings.Split(issueValue.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if issueValue.DataSource == 1 { + years = toolYears + } else if issueValue.DataSource == 3 { + years = manYears + } + if cveYears <= years { + icr := models.IssueCreateRecord{} + models.UpdateIssueStatus(issueValue, 4) + logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", + years, ",否则不需要在git上提交issue, cveData: ", issueValue) + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + continue + } + GenIssueRecordData(&icr, issueValue, sc, 2) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } + continue + } + } + } + // Determine whether cve has been processed + issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) + if issueExist { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) + continue + } + // Process each piece of cve data + if issueValue.Status == 0 { + mutex.Lock() + err := ProcIssue(issueValue, accessToken, owner, path) + mutex.Unlock() + if err != nil { + logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } else { + mutex.Lock() + err := ProcUpdateIssue(issueValue, accessToken, owner, path) + mutex.Unlock() + if err != nil { + logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } + // Collect issue record data + icr := models.IssueCreateRecord{} + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + continue + } + GenIssueRecordData(&icr, issueValue, sc, 1) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } + } + } + } + } else { + logs.Error(issueErr) + return issueErr + } + return nil +} + //CreateIssue Create issue func CreateIssue() error { defer common.Catchs() @@ -115,204 +331,14 @@ func CreateIssue() error { } beforeTime := common.GetBeforeTime(days) if issueWhitelist == 1 { - issueWhiteData, issueErr := models.QueryIssueWhitelist(1) - if issueErr == nil && len(issueWhiteData) > 0 { - for _, issues := range issueWhiteData { - for ; ; { - cveData, err := models.QueryIssueByPackName(issues.PackageName, beforeTime, prcnum) - if err == nil && len(cveData) > 0 { - logs.Info(cveData) - } else { - logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) - break - } - accessToken := os.Getenv("GITEE_TOKEN") - if accessToken == "" || len(accessToken) < 1 { - logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime(), ",err: ", err) - break - } - owner := BConfig.String("gitee::owner") - path := BConfig.String("gitee::path") - for index, issueValue := range cveData { - logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) - // Determine whether the issue has been processed - goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) - if oks { - if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || - goi.State == "已完成" || goi.State == "已拒绝" { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) - continue - } - } - // Import cve as data after 2018 - cveNumList := strings.Split(issueValue.CveNum, "-") - if cveNumList != nil && len(cveNumList) > 1 { - cveYears, yearErr := strconv.Atoi(cveNumList[1]) - if yearErr == nil { - if issueValue.DataSource == 1 { - years = toolYears - } else if issueValue.DataSource == 3 { - years = manYears - } - if cveYears <= years { - icr := models.IssueCreateRecord{} - models.UpdateIssueStatus(issueValue, 4) - logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", - years, ",否则不需要在git上提交issue, cveData: ", issueValue) - sc, err := models.QueryIssueScore(issueValue.CveId) - if err != nil || sc.Id == 0 { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) - continue - } - GenIssueRecordData(&icr, issueValue, sc, 2) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) - } - continue - } - } - } - // Determine whether cve has been processed - issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) - if issueExist { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) - continue - } - // Process each piece of cve data - if issueValue.Status == 0 { - err := ProcIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) - continue - } - } else { - err := ProcUpdateIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) - continue - } - } - // Collect issue record data - icr := models.IssueCreateRecord{} - sc, err := models.QueryIssueScore(issueValue.CveId) - if err != nil || sc.Id == 0 { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) - continue - } - GenIssueRecordData(&icr, issueValue, sc, 1) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) - } - } - } - } - } + limitErr := addLimitedIssue(beforeTime, prcnum, years, toolYears, manYears) + logs.Error("addLimitedIssue, err: ", limitErr) + return limitErr } else { - for ; ; { - cveData, err := models.QueryIssue(beforeTime, prcnum) - if err == nil && len(cveData) > 0 { - logs.Info(cveData) - } else { - logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) - return err - } - accessToken := os.Getenv("GITEE_TOKEN") - if accessToken == "" || len(accessToken) < 1 { - logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) - return err - } - owner := BConfig.String("gitee::owner") - path := BConfig.String("gitee::path") - for index, issueValue := range cveData { - logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) - // Determine whether the issue has been processed - goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) - if oks { - if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || - goi.State == "已完成" || goi.State == "已拒绝" { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) - continue - } - } - // Import cve as data after 2018 - cveNumList := strings.Split(issueValue.CveNum, "-") - if cveNumList != nil && len(cveNumList) > 1 { - cveYears, yearErr := strconv.Atoi(cveNumList[1]) - if yearErr == nil { - if issueValue.DataSource == 1 { - years = toolYears - } else if issueValue.DataSource == 3 { - years = manYears - } - if cveYears <= years { - icr := models.IssueCreateRecord{} - models.UpdateIssueStatus(issueValue, 4) - logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", - years, ",否则不需要在git上提交issue, cveData: ", issueValue) - sc, err := models.QueryIssueScore(issueValue.CveId) - if err != nil || sc.Id == 0 { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) - continue - } - GenIssueRecordData(&icr, issueValue, sc, 2) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) - } - continue - } - } - } - // Determine whether cve has been processed - issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) - if issueExist { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) - continue - } - // Process each piece of cve data - if issueValue.Status == 0 { - err := ProcIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) - continue - } - } else { - err := ProcUpdateIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) - continue - } - } - // Collect issue record data - icr := models.IssueCreateRecord{} - sc, err := models.QueryIssueScore(issueValue.CveId) - if err != nil || sc.Id == 0 { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) - continue - } - GenIssueRecordData(&icr, issueValue, sc, 1) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) - } - } - } + unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears) + logs.Error("addUnlimitedIssue, err: ", unlimitErr) + return unlimitErr } - return nil } //ProcUpdateIssue Update issue diff --git a/taskhandler/common.go b/taskhandler/common.go index 45809fe51d429db371f8cf912f14946efd3ca3e6..2512990a19ef0352ef0d765fada9a227bffd3a23 100644 --- a/taskhandler/common.go +++ b/taskhandler/common.go @@ -74,7 +74,7 @@ const commentCopyValue = ` %v **issue处理注意事项:** **1. 当前issue受影响的分支提交pr时, 须在pr描述中填写当前issue编号进行关联, 否则无法关闭当前issue;** -**2. 模板内容需要填写完整, 如果是非maintainer填写, 需要maintainer审核通过, 否则无法关闭当前issue;** +**2. 模板内容需要填写完整, 无论是受影响或者不受影响都需要填写完整内容, 否则无法关闭当前issue;** **3. 以下为模板中需要填写完整的内容, 请复制到评论区回复, 注: 内容的标题名称(影响性分析说明, openEuler评分, 受影响版本排查(受影响/不受影响))不能省略,省略后cve-manager将无法正常解析填写内容.** ************************************************************************ 影响性分析说明: diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index 5ad92c9156a0508af03526ce2caca5cd33abbe6f..dec46635a67835d03aa49b488666adc56685ad76 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -12,8 +12,11 @@ import ( "github.com/astaxie/beego/logs" "strconv" "strings" + "sync" ) +var updateLock sync.Mutex + func CreateIssueData(issueTemp *models.IssueTemplate, cve models.VulnCenter, sc models.Score, resp map[string]interface{}, path, assignee, issueType, labels, owner string) *models.IssueTemplate { issueTemp.CveId = cve.CveId @@ -92,6 +95,12 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st var it models.IssueTemplate it.CveId = cve.CveId _, err := models.GetIssueTemplet(&it) + sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} + specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + it.Assignee = sia.Assignee + assignee = sia.Assignee + } if err && it.IssueNum != "" && len(it.IssueNum) > 0 { if it.Assignee == "" || len(it.Assignee) == 0 { it.Assignee = assignee @@ -265,6 +274,11 @@ func UpdateIssueToGit(accessToken string, owner string, path string, cve models.VulnCenter, its models.IssueTemplate) (string, error) { logs.Info("更新模板请求参数: cve: ", cve, ",its: ", its, ", owner: ", owner, ",path: ", path) //labels := its.IssueLabel + sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} + specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + its.Assignee = sia.Assignee + } labels := "" if its.IssueLabel != "" && len(its.IssueLabel) > 1 { labels = its.IssueLabel @@ -292,7 +306,9 @@ func UpdateIssueToGit(accessToken string, owner string, path string, cve, sc, OpenEulerScore, score, labels, its, 3, its.IssueType, pkgLink, brandArray) logs.Info("isssue_body: ", requestBody) if requestBody != "" && len(requestBody) > 1 { + updateLock.Lock() resp, err := util.HTTPPatch(url, requestBody) + updateLock.Unlock() if err != nil { logs.Error("更新issue失败, cveNum: ", cve.CveNum, "err: ", err) return "", err diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 06a47a9e9d03d3bfc39450c570b2165d97085060..390aec82f4e10b4c46ac2300704912395d1cc6b2 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -1695,7 +1695,7 @@ func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { du := "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" //du := beego.AppConfig.String("excel::v_pack_20_03_url") du = beego.AppConfig.DefaultString("excel::v_pack_20_03_url", du) - localPath := filepath.Join(dir, "release-package.CSV") + localPath := filepath.Join(dir, affectBranch + "_release-package.CSV") err = downloadPackageFile(localPath, du) if err != nil { logs.Error(err) diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 30f10c741fe28dd087d11b125a699095fcd5b797..8fb1bbacb12fb76ed9f9cafa02ee38ca8aea18be 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -775,7 +775,6 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I if ok != nil { saTimeStampZone = 3600 * 8 } - //owner := "src-openeuler" st := util.TimeStrToInt(startTime, "2006-01-02") chData := make([]IssueAndPkg, 0) for _, v := range pkgList { diff --git a/util/http.go b/util/http.go index 70007aadfe4a68d985869675054de96d49d00973..345c28807e387cd638d6741a3a3b3d56dca3960b 100644 --- a/util/http.go +++ b/util/http.go @@ -2,6 +2,7 @@ package util import ( "bytes" + "cvevulner/common" "encoding/json" "fmt" "github.com/astaxie/beego/logs" @@ -22,6 +23,7 @@ type RequestInfo struct { //HTTPPatch patch request func HTTPPatch(url string, requestBody string) (map[string]interface{}, error) { req, err := http.NewRequest("PATCH", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() if err != nil { return nil, err } @@ -60,6 +62,7 @@ func HTTPPatch(url string, requestBody string) (map[string]interface{}, error) { //HTTPPost post request func HTTPPost(url string, requestBody string) (map[string]interface{}, error) { req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() if err != nil { return nil, err } @@ -98,6 +101,7 @@ func HTTPPost(url string, requestBody string) (map[string]interface{}, error) { //HTTPPost1 post request func HTTPPost1(url string, requestBody string) ([]map[string]interface{}, error) { req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() if err != nil { return nil, err }