diff --git a/common/common.go b/common/common.go index 37f54c43f8c0eb04a71045e946316636d0dd1d84..c1218f71a98b6d94a596fc468a3b2bb8ca3fd50e 100644 --- a/common/common.go +++ b/common/common.go @@ -10,6 +10,7 @@ import ( ) func DesString(dbpwd string) (strs string) { + defer Catchs() if "" != dbpwd || len(dbpwd) > 0 { key := beego.AppConfig.String("key") key1 := []byte(key) @@ -21,6 +22,24 @@ func DesString(dbpwd string) (strs string) { return strs } +func AesString(dbpwd string) (strs string) { + defer Catchs() + pwd := []byte{} + if dbpwd == "" { + pwd = []byte("123") + } else { + pwd = []byte(dbpwd) + } + key := []byte(beego.AppConfig.String("key")) + strs, err := EnPwdCode(pwd, key) + if err != nil { + logs.Error(err) + }else { + logs.Info(strs) + } + return strs +} + func GetTokenExpirTime() (token_expir_time int) { token_expir_time, err := beego.AppConfig.Int("token_expir_time") if err == nil { diff --git a/conf/app.conf b/conf/app.conf index c11b18fdae354ad12f74fae61d4e32b4c378af5c..b25442fdfd66343061623545b7b340ccc72a3f8e 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -61,7 +61,7 @@ genexcelflag = 2 genexcel = 0 */10 * * * * days = -30 prcnum = 50 -printlogflag = 2 +printlogflag = 1 printlog = 0 */20 * * * * @@ -103,6 +103,10 @@ cveref = https://nvd.nist.gov/vuln/detail/ openeulernum = 3000 # Import cve as data after 2018 cve_number = 2018 +# Manual data import restrictions +cve_number_m = 1990 +# Tool import data restriction +cve_number_t = 2018 # Create an issue's warehouse whitelist;1: open; 2: close issue_whitelist = 2 # List of affected branches diff --git a/conf/product_app.conf b/conf/product_app.conf index 58da96caf1874cf6f74ec07db726a9443b74f831..49c0aebbcb0e82bb33587fa38916c8f0bc1cac37 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -99,12 +99,16 @@ cveref = https://nvd.nist.gov/vuln/detail/ openeulernum = 3000 # Import cve as data after 2018 cve_number = 2018 +# Manual data import restrictions +cve_number_m = 1990 +# Tool import data restriction +cve_number_t = 2018 # Create an issue's warehouse whitelist;1: open; 2: close issue_whitelist = 1 # List of affected branches affected_branchs = openEuler-20.03-LTS # Close the highest privilege of issue -close_issue_privilege = 2 +close_issue_privilege = 1 [reflink] diff --git a/controllers/cvedetail.go b/controllers/cvedetail.go index a8446551dbb2ea64b12ef928970ee24d5e01e723..afa83d002d933ce0835627fdf92af6e3f922b248 100644 --- a/controllers/cvedetail.go +++ b/controllers/cvedetail.go @@ -35,6 +35,25 @@ func (u *CveDetailController) Get() { resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) resp["body"] = cod defer u.RetData(resp) + //Judge whether it is legal + token := u.GetString("token") + if token == "" { + resp["errno"] = errcode.RecodeSessionErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) + resp["body"] = []ResultData{} + logs.Error("token 校验失败") + return + } else { + // Check token + ok := models.CheckToken(token) + if !ok { + resp["errno"] = errcode.RecodeSessionErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) + resp["body"] = []ResultData{} + logs.Error("token 校验失败") + return + } + } cveNum := u.GetString("cveNum") if cveNum == "" { logs.Error("cveNum, 参数错误") diff --git a/controllers/login.go b/controllers/login.go index ae3757af986b6a9039cc1f08539a37729b3b5c0d..848129916929a642da0b65b794328a2bdbd51661 100644 --- a/controllers/login.go +++ b/controllers/login.go @@ -43,7 +43,7 @@ func (u *UserLoginController) Post() { logs.Error(err) return } - logs.Info("登录请求参数：", &req) + //logs.Info("登录请求参数：", req) //Judge whether it is legal if req["userName"] == nil || req["passWord"] == nil { resp["errno"] = errcode.RecodeDataErr @@ -54,7 +54,7 @@ func (u *UserLoginController) Post() { } password := fmt.Sprintf("%s", req["passWord"]) // Encryption first comment - //password = common.DesString(password) + password = common.DesString(password) if password == "" || len(password) == 0 { resp["errno"] = errcode.RecodePwdErr resp["errmsg"] = errcode.RecodeText(errcode.RecodePwdErr) diff --git a/cve-py/controller/taskcontroller.py b/cve-py/controller/taskcontroller.py index 0431de82af145aa86ada62a0ffa5d22f16f2aee9..d0d5d269efc9c2e16750b1a30a404b7ee872e43a 100644 --- a/cve-py/controller/taskcontroller.py +++ b/cve-py/controller/taskcontroller.py @@ -13,7 +13,7 @@ Authors: xiaojianghui Date: 10/22/2020 11:01 AM """ -from tabletask import runtask, mappingtask, toexcel, export_excel_task, import_excel_task +from tabletask import runtask, mappingtask, toexcel, export_excel_task, import_excel_task, package_committer_task from gitwebtask import genegroup, yamltask from emailtask import sendingtask, issue_record_email from deletetask import deletefiletask @@ -107,3 +107,14 @@ def run_whiltlist_task(): print('the task of parsing package whitelist start') import_excel_task.import_data() print('the task of parsing package whitelist complete') + + +def run_package_committer_task(): + """ + the task of parsing package_committer excels + Returns:None + + """ + print('the task of parsing package_committer_excels start') + package_committer_task.import_data() + print('the task of parsing package_committer_excels complete') diff --git a/cve-py/controller/timertaskcontroller.py b/cve-py/controller/timertaskcontroller.py index e18c33f0b4877f16f18a252c5f9d378a2a1b45f4..a72572126d048d0ce972e4d0717a30b2b37e12ef 100644 --- a/cve-py/controller/timertaskcontroller.py +++ b/cve-py/controller/timertaskcontroller.py @@ -39,6 +39,8 @@ def timertask(): scheduler.add_job(taskcontroller.run_issue_record_task, 'cron', day_of_week='0-6', hour=7, minute=30) # analysis the table package whitelist scheduler.add_job(taskcontroller.run_whiltlist_task, 'interval', hours=2) + # parsing the table package_committer_excels + scheduler.add_job(taskcontroller.run_package_committer_task, 'interval', hours=1) scheduler.start() except SystemExit as err: print("Err:", err) diff --git a/cve-py/downloadtask/downloadfiletask.py b/cve-py/downloadtask/downloadfiletask.py index b2667a3d6a49af0f808fef1e1ba22df09e68be89..c91c82de7d9e807265ccb9d43114febfe54af162 100644 --- a/cve-py/downloadtask/downloadfiletask.py +++ b/cve-py/downloadtask/downloadfiletask.py @@ -123,3 +123,38 @@ def handle_three(): if os.path.exists("./import_excels/" + name): print("Download the Package whitelist table successfully:" + name) i += 2 + + +def handle_four(): + """ + download file + """ + if os.listdir("./package_committer_excels"): + shutil.rmtree("./package_committer_excels") + os.mkdir("./package_committer_excels") + url = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/package_committer_excels' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/package_committer_excels/" + name + r = requests.get(url) + print(r.status_code) + with open("./package_committer_excels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./package_committer_excels/" + name): + print("Download the package_committer table successfully:" + name) + i += 2 diff --git a/cve-py/import_excels/266_whitelist_2020-11-26.xlsx b/cve-py/import_excels/266_whitelist_2020-11-26.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..9691d07170b150faca2ecd73c828bd20724c23d1 Binary files /dev/null and b/cve-py/import_excels/266_whitelist_2020-11-26.xlsx differ diff --git a/cve-py/import_excels/21_whitelist_2020-11-21.xlsx b/cve-py/import_excels/26_whitelist_2020-11-26.xlsx similarity index 79% rename from cve-py/import_excels/21_whitelist_2020-11-21.xlsx rename to cve-py/import_excels/26_whitelist_2020-11-26.xlsx index 98b98e3761efd81dc098112c10f1450dfc6cd76c..98eca1855bd0e96b11792afdb999d92da80845ca 100644 Binary files a/cve-py/import_excels/21_whitelist_2020-11-21.xlsx and b/cve-py/import_excels/26_whitelist_2020-11-26.xlsx differ diff --git a/cve-py/newexcels/20.03-CVE_2020-11-26.xlsx b/cve-py/newexcels/20.03-CVE_2020-11-26.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..1ed659b011ec0c6d8c825d6b6396ed695064c4e4 Binary files /dev/null and b/cve-py/newexcels/20.03-CVE_2020-11-26.xlsx differ diff --git a/cve-py/newexcels/cve_issue_2020-11-26.xlsx b/cve-py/newexcels/cve_issue_2020-11-26.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..f7c12d6140def0803a44d82e1b1fd66a79ccc980 Binary files /dev/null and b/cve-py/newexcels/cve_issue_2020-11-26.xlsx differ diff --git "a/cve-py/package_committer_excels/openEuler\347\244\276\345\214\272\350\275\257\344\273\266\345\214\205\350\264\243\344\273\273\347\224\260-openeuler.xlsx" "b/cve-py/package_committer_excels/openEuler\347\244\276\345\214\272\350\275\257\344\273\266\345\214\205\350\264\243\344\273\273\347\224\260-openeuler.xlsx" new file mode 100644 index 0000000000000000000000000000000000000000..efdab9447513e025e5efa1d38ef821f9ca36b3b3 Binary files /dev/null and "b/cve-py/package_committer_excels/openEuler\347\244\276\345\214\272\350\275\257\344\273\266\345\214\205\350\264\243\344\273\273\347\224\260-openeuler.xlsx" differ diff --git a/cve-py/tabletask/package_committer_task.py b/cve-py/tabletask/package_committer_task.py new file mode 100644 index 0000000000000000000000000000000000000000..c36cdfc1fe2987ba1dcb26f24ea0876a7e2ecb68 --- /dev/null +++ b/cve-py/tabletask/package_committer_task.py @@ -0,0 +1,81 @@ +#!/usr/bin/python3 +# -*- coding: UTF-8 -*- + + +import os +import xlrd +import time +import hashlib +from dbConnecttion.MysqlConn import Mysql +from downloadtask import downloadfiletask + + +def parse_excel(file_name): + """ + parse excel data + parse: file_name + return: list + """ + package_name_list = [] + issue_assignee_list = [] + try: + data = xlrd.open_workbook('./package_committer_excels/' + file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + package_name_list.append(table_one.cell(i, 0).value) + issue_assignee_list.append(table_one.cell(i, 1).value) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return package_name_list, issue_assignee_list + + +def cur_date(): + """ + current date + :return createTime: string + """ + create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + return create_time + + +def import_data(): + """ + import excel + :param + :return none + """ + downloadfiletask.handle_four() + mysql = Mysql() + files = os.listdir('./package_committer_excels') + for filename in files: + with open('./package_committer_excels/' + filename, 'rb') as f: + sha1obj = hashlib.sha1() + sha1obj.update(f.read()) + hash_value = sha1obj.hexdigest() + print(filename, hash_value) + f.close() + sql = "select file_hash from cve_file_hash where file_name = %s" + val = (filename,) + file_hash = mysql.getOne(sql, val) + if file_hash: + if hash_value == file_hash['file_hash']: + print("文件已解析:" + filename) + os.remove('./package_committer_excels/' + filename) + continue + sql = "insert into cve_file_hash (file_name, file_hash) values (%s, %s)" + val = (filename, hash_value) + mysql.insertOne(sql, val) + mysql.dispose() + result = parse_excel(filename) + for i in range(0, len(result[0])): + sql = 'insert ignore into cve_spec_issue_assigness (package_name, issue_assignee, status, create_time) ' \ + 'values (%s, %s, %s, %s)' + val = (result[0][i], result[1][i], 1, cur_date()) + mysql.insertOne(sql, val) + mysql.dispose() + print('插入数据{}:{}'.format(result[0][i], result[1][i])) + os.remove('./package_committer_excels/' + filename) + mysql.close() diff --git a/doc/sql/db_struct.sql b/doc/sql/db_struct.sql index 420bfe82b5d79ea2d57bfd4e4a23c8a0056d63a5..e3b03c85221c151e4a91ae1b58709c63c835ba33 100644 --- a/doc/sql/db_struct.sql +++ b/doc/sql/db_struct.sql @@ -52,7 +52,18 @@ CREATE TABLE `cve_export_record` ( `state` tinyint NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `file_name` (`file_name`) -) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=55 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_file_hash` */ + +DROP TABLE IF EXISTS `cve_file_hash`; + +CREATE TABLE `cve_file_hash` ( + `id` int NOT NULL AUTO_INCREMENT, + `file_name` varchar(50) DEFAULT NULL, + `file_hash` varchar(50) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_open_euler` */ @@ -86,7 +97,7 @@ CREATE TABLE `cve_git_open_euler` ( KEY `cve_git_open_euler_package_id` (`package_id`), KEY `cve_git_open_euler_version` (`version`), KEY `cve_git_open_euler_table_id` (`table_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4748 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=5071 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_open_euler_table_relate` */ @@ -241,7 +252,7 @@ CREATE TABLE `cve_gite_origin_issue` ( PRIMARY KEY (`id`), UNIQUE KEY `issue_id` (`issue_id`), UNIQUE KEY `number` (`number`) -) ENGINE=InnoDB AUTO_INCREMENT=591 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=4190 DEFAULT CHARSET=utf8; /*Table structure for table `cve_gite_repo` */ @@ -262,7 +273,7 @@ CREATE TABLE `cve_gite_repo` ( `delete_time` varchar(255) DEFAULT NULL, PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_group_id` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=28649 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=28651 DEFAULT CHARSET=utf8; /*Table structure for table `cve_gite_repo_branch` */ @@ -293,7 +304,7 @@ CREATE TABLE `cve_gite_repo_member` ( PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_member_group_id` (`group_id`), CONSTRAINT `member_groups_group_id` FOREIGN KEY (`group_id`) REFERENCES `cve_git_repo_groups` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1766 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=1768 DEFAULT CHARSET=utf8; /*Table structure for table `cve_ip_white` */ @@ -317,7 +328,31 @@ CREATE TABLE `cve_issue_assignee` ( `assignee` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY (`id`), UNIQUE KEY `assignee` (`assignee`) -) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_create_record` */ + +DROP TABLE IF EXISTS `cve_issue_create_record`; + +CREATE TABLE `cve_issue_create_record` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `cve_id` bigint NOT NULL DEFAULT '0', + `cve_num` varchar(256) NOT NULL DEFAULT '', + `cve_desc` varchar(8192) NOT NULL DEFAULT '', + `cve_level` varchar(32) NOT NULL DEFAULT '', + `cve_version` varchar(128) NOT NULL DEFAULT '', + `repair_time` varchar(32) NOT NULL DEFAULT '', + `pack_name` varchar(512) NOT NULL DEFAULT '', + `nvd_score` decimal(10,1) NOT NULL DEFAULT '0.0', + `n_vector_value` varchar(256) NOT NULL DEFAULT '', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + `status` tinyint NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + KEY `cve_issue_create_record_cve_id` (`cve_id`), + KEY `cve_issue_create_record_cve_num` (`cve_num`) +) ENGINE=InnoDB AUTO_INCREMENT=1425 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_hooks` */ @@ -345,7 +380,25 @@ CREATE TABLE `cve_issue_hooks` ( UNIQUE KEY `cve_issue_hooks_owner_repo_status_un` (`owner`,`repo`,`hook_url`,`status`), KEY `cve_issue_hooks_Cve_id` (`cve_id`), KEY `cve_issue_hooks_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=21 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=27 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_issue_repo_whitelist` */ + +DROP TABLE IF EXISTS `cve_issue_repo_whitelist`; + +CREATE TABLE `cve_issue_repo_whitelist` ( + `repo_id` bigint NOT NULL AUTO_INCREMENT, + `package_name` varchar(256) NOT NULL DEFAULT '', + `version` varchar(64) NOT NULL DEFAULT '', + `branchs` varchar(512) DEFAULT NULL, + `status` tinyint NOT NULL DEFAULT '0', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`repo_id`), + KEY `cve_issue_repo_whitelist_package_name` (`package_name`), + KEY `cve_issue_repo_whitelist_version` (`version`) +) ENGINE=InnoDB AUTO_INCREMENT=143 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_template` */ @@ -387,11 +440,15 @@ CREATE TABLE `cve_issue_template` ( `update_time` datetime NOT NULL, `delete_time` datetime NOT NULL, `error_description` text CHARACTER SET utf8 COLLATE utf8_general_ci, + `mt_audit_flag` tinyint NOT NULL DEFAULT '0', + `sa_audit_flag` tinyint NOT NULL DEFAULT '0', + `op_audit_flag` tinyint NOT NULL DEFAULT '0', PRIMARY KEY (`template_id`), UNIQUE KEY `cve_issue_template_cvn_issnu` (`cve_num`,`issue_num`), + UNIQUE KEY `cve_issue_template_num_compone_versio` (`cve_num`,`owned_component`,`owned_version`), KEY `cve_issue_template_cve_id` (`cve_id`), KEY `cve_issue_template_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=625 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2051 DEFAULT CHARSET=utf8; /*Table structure for table `cve_open_euler_s_a` */ @@ -405,7 +462,25 @@ CREATE TABLE `cve_open_euler_s_a` ( PRIMARY KEY (`openeuler_id`), UNIQUE KEY `openeuler_sa_num` (`openeuler_sa_num`), KEY `cve_open_euler_s_a_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1949 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2676 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_open_guss_yaml` */ + +DROP TABLE IF EXISTS `cve_open_guss_yaml`; + +CREATE TABLE `cve_open_guss_yaml` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `package_name` varchar(256) NOT NULL DEFAULT '', + `version` varchar(64) NOT NULL DEFAULT '', + `origin_url` varchar(512) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `package_name_version` (`package_name`,`version`), + KEY `cve_open_guss_yaml_version` (`version`) +) ENGINE=InnoDB AUTO_INCREMENT=631 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_excel` */ @@ -440,7 +515,7 @@ CREATE TABLE `cve_origin_excel` ( `delete_time` datetime DEFAULT NULL, PRIMARY KEY (`cve_id`) USING BTREE, UNIQUE KEY `cve_origin_excel_num_un` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; +) ENGINE=InnoDB AUTO_INCREMENT=235 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; /*Table structure for table `cve_origin_upstream` */ @@ -473,7 +548,7 @@ CREATE TABLE `cve_origin_upstream` ( KEY `cve_origin_upstream_cve_packname` (`cve_packname`), KEY `cve_origin_upstream_git_packname` (`git_packname`), KEY `cve_origin_upstream_version` (`version`) -) ENGINE=InnoDB AUTO_INCREMENT=1457 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=1847 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config` */ @@ -485,7 +560,7 @@ CREATE TABLE `cve_origin_upstream_config` ( `nodes` varchar(32) DEFAULT NULL, PRIMARY KEY (`conf_id`), KEY `cve_origin_upstream_config_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node` */ @@ -497,7 +572,7 @@ CREATE TABLE `cve_origin_upstream_config_node` ( `operator` varchar(256) DEFAULT NULL, PRIMARY KEY (`node_id`), KEY `cve_origin_upstream_config_node_conf_id` (`conf_id`) -) ENGINE=InnoDB AUTO_INCREMENT=8024 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=24801 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node_cpe` */ @@ -511,7 +586,7 @@ CREATE TABLE `cve_origin_upstream_config_node_cpe` ( `vulner_able` varchar(64) DEFAULT NULL, PRIMARY KEY (`cpe_id`), KEY `cve_origin_upstream_config_node_cpe_node_id` (`node_id`) -) ENGINE=InnoDB AUTO_INCREMENT=246346 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=767810 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_desc` */ @@ -524,7 +599,7 @@ CREATE TABLE `cve_origin_upstream_desc` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`desc_id`), KEY `cve_origin_upstream_desc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_event` */ @@ -539,7 +614,7 @@ CREATE TABLE `cve_origin_upstream_event` ( `description` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`event_id`), KEY `cve_origin_upstream_event_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest` */ @@ -551,7 +626,7 @@ CREATE TABLE `cve_origin_upstream_fix_suggest` ( `detail` varchar(1024) DEFAULT NULL, PRIMARY KEY (`fix_id`), KEY `cve_origin_upstream_fix_suggest_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4219 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13125 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest_ref` */ @@ -589,7 +664,7 @@ CREATE TABLE `cve_origin_upstream_impact` ( `impact` varchar(32) DEFAULT NULL, PRIMARY KEY (`impact_id`), KEY `cve_origin_upstream_impact_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score` */ @@ -605,7 +680,7 @@ CREATE TABLE `cve_origin_upstream_impact_score` ( `score_status` tinyint DEFAULT '1', PRIMARY KEY (`score_id`), KEY `cve_origin_upstream_impact_score_impact_id` (`impact_id`) -) ENGINE=InnoDB AUTO_INCREMENT=8473 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=26355 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v2` */ @@ -634,7 +709,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v2` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v2_id`), KEY `cve_origin_upstream_impact_score_v2_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v3` */ @@ -660,7 +735,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v3` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v3_id`), KEY `cve_origin_upstream_impact_score_v3_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_poc` */ @@ -677,7 +752,7 @@ CREATE TABLE `cve_origin_upstream_poc` ( `desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`poc_id`), KEY `cve_origin_upstream_poc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4237 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_reference` */ @@ -692,7 +767,7 @@ CREATE TABLE `cve_origin_upstream_reference` ( `tags` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`ref_id`), KEY `cve_origin_upstream_reference_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=83782 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=260700 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_vul_type` */ @@ -706,7 +781,7 @@ CREATE TABLE `cve_origin_upstream_vul_type` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`vul_id`), KEY `cve_origin_upstream_vul_type_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=4219 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13125 DEFAULT CHARSET=utf8; /*Table structure for table `cve_other_user` */ @@ -722,7 +797,7 @@ CREATE TABLE `cve_other_user` ( `update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, `delete_time` datetime DEFAULT NULL, PRIMARY KEY (`user_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; /*Table structure for table `cve_package` */ @@ -735,7 +810,7 @@ CREATE TABLE `cve_package` ( `pack_url` varchar(2048) DEFAULT NULL, PRIMARY KEY (`id`), KEY `cve_package_sec_id` (`sec_id`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=10087 DEFAULT CHARSET=utf8; /*Table structure for table `cve_package_cpe` */ @@ -794,7 +869,7 @@ CREATE TABLE `cve_score` ( PRIMARY KEY (`id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_score_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1949 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2676 DEFAULT CHARSET=utf8; /*Table structure for table `cve_score_record` */ @@ -809,7 +884,7 @@ CREATE TABLE `cve_score_record` ( `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `cve_score_record_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1951 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3450 DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_notice` */ @@ -835,7 +910,7 @@ CREATE TABLE `cve_security_notice` ( PRIMARY KEY (`sec_id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_security_notice_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1949 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2676 DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_reviewer` */ @@ -847,7 +922,23 @@ CREATE TABLE `cve_security_reviewer` ( `status` tinyint NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `name_space` (`name_space`) -) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_spec_issue_assigness` */ + +DROP TABLE IF EXISTS `cve_spec_issue_assigness`; + +CREATE TABLE `cve_spec_issue_assigness` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `package_name` varchar(256) NOT NULL DEFAULT '', + `issue_assignee` varchar(256) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `cve_spec_issue_assigness_package_name` (`package_name`) +) ENGINE=InnoDB AUTO_INCREMENT=1195 DEFAULT CHARSET=utf8; /*Table structure for table `cve_vuln_center` */ @@ -872,7 +963,7 @@ CREATE TABLE `cve_vuln_center` ( PRIMARY KEY (`cve_id`), UNIQUE KEY `cve_vuln_center_num_pack_v_un` (`cve_num`,`cve_version`,`pack_name`), KEY `cve_vuln_center_cve_num` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=1955 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2682 DEFAULT CHARSET=utf8; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; diff --git a/models/issue.go b/models/issue.go index 2d757b4991b77ae4ff403432e3d04f6bc7db31c4..a08b8409835f573a261a76f99a416565da476eb7 100644 --- a/models/issue.go +++ b/models/issue.go @@ -31,8 +31,8 @@ func QueryIssueWhitelist(status int8) ([]IssueRepoWhitelist, error) { o := orm.NewOrm() var irw []IssueRepoWhitelist if status == 0 { - num, err := o.Raw("select repo_id,package_name,version,branchs,status,create_time,"+ - "update_time,delete_time"+ + num, err := o.Raw("select repo_id,package_name,version,branchs,status,create_time," + + "update_time,delete_time" + " from cve_issue_repo_whitelist order by repo_id desc").QueryRows(&irw) if err == nil && num > 0 { logs.Info("cve_issue_repo_whitelist 查询结果：", irw) @@ -59,7 +59,7 @@ func QueryIssueByPackName(packName, days string, prcnum int) ([]VulnCenter, erro var vc []VulnCenter num, err := o.Raw("select cve_id,cve_num,cve_desc,cve_version,repair_time,"+ "pack_name,cve_url,cve_level,data_source,update_time,is_export,cve_detail_url"+ - " from cve_vuln_center where pack_name = ? and " + + " from cve_vuln_center where pack_name = ? and "+ "update_time >= ? and cve_status in (?, ?) "+ "order by cve_id asc limit ?", packName, days, 0, 1, prcnum).QueryRows(&vc) if err == nil && num > 0 { @@ -175,6 +175,12 @@ func GetIssueTemplateByColName(it *IssueTemplate, colName ...string) error { return err } +func GetSpecIssueAssignee(sia *SpecIssueAssigness, colName ...string) error { + o := orm.NewOrm() + err := o.Read(sia, colName...) + return err +} + func UpdateIssueTemplate(it *IssueTemplate, fields ...string) error { logs.Info("===>", it, fields) o := orm.NewOrm() @@ -247,7 +253,7 @@ func ReplacePackageByCveId(pkgList []string, cveId int64) error { platform := "source" if strings.Contains(v, ".x86_64.") { platform = "x86_64" - } else if strings.Contains(v, ".aarch64.") || strings.Contains(v, ".aarch64.") { + } else if strings.Contains(v, ".aarch64.") || strings.Contains(v, ".noarch.") { platform = "aarch64" } pkgUrl := fmt.Sprintf(`https://repo.openeuler.org/openEuler-20.03-LTS/update/%s/Packages/%s`, platform, v) @@ -268,7 +274,7 @@ func QueryPackageByCveId(cveId int64) ([]Package, error) { func InsertIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { o := orm.NewOrm() - if issTempId, err = o.Insert(it); err == nil && issTempId > 0 { + if issTempId, err = o.Insert(it); err == nil && issTempId > 0 { logs.Info("insert cve_issue_template success, issTempId: ", issTempId, "cveNum: ", it.CveNum) } else { logs.Error("insert cve_issue_template failed, cveNum:", it.CveNum, "err: ", err) @@ -477,4 +483,3 @@ func CreateIssueRecord(icr *IssueCreateRecord) (issueRecordId int64, err error) return icr.IssueRecordId, nil } } - diff --git a/models/modeldb.go b/models/modeldb.go index de080590b4c5874a003a6154cfbd91f51c03024c..cc1258ff35d3e19fe3b58f7c135914ede5159592 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -604,6 +604,27 @@ type IssueCreateRecord struct { DeleteTime string `orm:"size(32);column(delete_time);null"` } +type OpenGussYaml struct { + Id int64 `orm:"pk;auto"` + PackageName string `orm:"column(package_name);size(256)" description:"包名称"` + Version string `orm:"size(64);column(version);index" description:"版本号"` + OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` + Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type SpecIssueAssigness struct { + Id int64 `orm:"pk;auto"` + PackageName string `orm:"column(package_name);size(256);unique" description:"包名称"` + Assignee string `orm:"size(256);column(issue_assignee);" description:"包对应的责任人"` + Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + func CreateDb() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -626,7 +647,7 @@ func CreateDb() bool { new(OriginUpstreamFixSuggest), new(OriginUpstreamFixSuggestRefTag), new(OriginUpstreamFixSuggestRef), new(GiteOriginIssue), new(OriginExcel), new(ExportRecord), new(GitRepoGroups), new(GiteRepo), new(GiteRepoMember), new(GiteRepoBranch), new(PackageCpe), new(EmailList), new(IssueAssignee), - new(IssueRepoWhitelist), new(IssueCreateRecord), + new(IssueRepoWhitelist), new(IssueCreateRecord), new(OpenGussYaml),new(SpecIssueAssigness), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/task/issuetask.go b/task/issuetask.go index 58d3032ab819ba32243f13319e9517bd235c0c43..bf2f95d24a349b407494476e1daf19a8818cddb0 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -11,6 +11,7 @@ import ( "os" "strconv" "strings" + "sync" ) //GetGiteeToken Get Code Cloud API TOKEN @@ -74,227 +75,272 @@ func GenIssueRecordData(icr *models.IssueCreateRecord, issueValue models.VulnCen icr.CreateTime = common.GetCurTime() } -//CreateIssue Create issue -func CreateIssue() error { - defer common.Catchs() - // Query the cve to be processed, 1: add; 2: modify - BConfig, err := config.NewConfig("ini", "conf/app.conf") - if err != nil { - logs.Error("config init error:", err) - return err - } - days, ok := BConfig.Int("crontab::days") - if ok != nil { - logs.Error("config crontab::days error:", ok) - return ok - } - years, ok := BConfig.Int("cve::cve_number") - if ok != nil { - logs.Error("config cve::years error:", ok) - return ok - } - prcnum, ok := BConfig.Int("crontab::prcnum") - if ok != nil { - logs.Error("config crontab::prcnum error:", ok) - return ok - } - issueWhitelist, ok := BConfig.Int("cve::issue_whitelist") - if ok != nil { - logs.Error("config cve::issue_whitelist error: ", ok) - return ok - } - beforeTime := common.GetBeforeTime(days) - if issueWhitelist == 1 { - issueWhiteData, issueErr := models.QueryIssueWhitelist(1) - if issueErr == nil && len(issueWhiteData) > 0 { - for _, issues := range issueWhiteData { - for ; ; { - cveData, err := models.QueryIssueByPackName(issues.PackageName, beforeTime, prcnum) - if err == nil && len(cveData) > 0 { - logs.Info(cveData) - } else { - logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) - break - } - accessToken := os.Getenv("GITEE_TOKEN") - if accessToken == "" || len(accessToken) < 1 { - logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime(), ",err: ", err) - break +var mutex sync.Mutex + +func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears int) error { + for ; ; { + cveData, err := models.QueryIssue(beforeTime, prcnum) + if err == nil && len(cveData) > 0 { + logs.Info(cveData) + } else { + logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) + return err + } + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) + return err + } + owner := beego.AppConfig.String("gitee::owner") + path := beego.AppConfig.String("gitee::path") + for index, issueValue := range cveData { + logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) + // Determine whether the issue has been processed + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝" { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) + continue + } + } + // Import cve as data after 2018 + cveNumList := strings.Split(issueValue.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if issueValue.DataSource == 1 { + years = toolYears + } else if issueValue.DataSource == 3 { + years = manYears } - owner := BConfig.String("gitee::owner") - path := BConfig.String("gitee::path") - for index, issueValue := range cveData { - logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) - // Determine whether the issue has been processed - goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) - if oks { - if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || - goi.State == "已完成" || goi.State == "已拒绝" { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) - continue - } - } - // Import cve as data after 2018 - cveNumList := strings.Split(issueValue.CveNum, "-") - if cveNumList != nil && len(cveNumList) > 1 { - cveYears, yearErr := strconv.Atoi(cveNumList[1]) - if yearErr == nil { - if cveYears <= years { - icr := models.IssueCreateRecord{} - models.UpdateIssueStatus(issueValue, 4) - logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", - years, ",否则不需要在git上提交issue, cveData: ", issueValue) - sc, err := models.QueryIssueScore(issueValue.CveId) - if err != nil || sc.Id == 0 { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) - continue - } - GenIssueRecordData(&icr, issueValue, sc, 2) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) - } - continue - } - } - } - // Determine whether cve has been processed - issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) - if issueExist { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) - continue - } - // Process each piece of cve data - if issueValue.Status == 0 { - err := ProcIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) - continue - } - } else { - err := ProcUpdateIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) - continue - } - } - // Collect issue record data + if cveYears <= years { icr := models.IssueCreateRecord{} + models.UpdateIssueStatus(issueValue, 4) + logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", + years, ",否则不需要在git上提交issue, cveData: ", issueValue) sc, err := models.QueryIssueScore(issueValue.CveId) if err != nil || sc.Id == 0 { logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) continue } - GenIssueRecordData(&icr, issueValue, sc, 1) + GenIssueRecordData(&icr, issueValue, sc, 2) issueRecordId, issReErr := models.CreateIssueRecord(&icr) if issReErr == nil && issueRecordId > 0 { logs.Info("Issue record data created successfully, id:", issueRecordId) } else { logs.Error("Failed to create issue record data, err: ", issReErr) } + continue } } } - } - } else { - for ; ; { - cveData, err := models.QueryIssue(beforeTime, prcnum) - if err == nil && len(cveData) > 0 { - logs.Info(cveData) + // Determine whether cve has been processed + issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) + if issueExist { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) + continue + } + // Process each piece of cve data + if issueValue.Status == 0 { + mutex.Lock() + err := ProcIssue(issueValue, accessToken, owner, path) + mutex.Unlock() + if err != nil { + logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } } else { - logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) - return err + mutex.Lock() + err := ProcUpdateIssue(issueValue, accessToken, owner, path) + mutex.Unlock() + if err != nil { + logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } } - accessToken := os.Getenv("GITEE_TOKEN") - if accessToken == "" || len(accessToken) < 1 { - logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) - return err + // Collect issue record data + icr := models.IssueCreateRecord{} + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + continue } - owner := BConfig.String("gitee::owner") - path := BConfig.String("gitee::path") - for index, issueValue := range cveData { - logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) - // Determine whether the issue has been processed - goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) - if oks { - if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || - goi.State == "已完成" || goi.State == "已拒绝" { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) - continue - } + GenIssueRecordData(&icr, issueValue, sc, 1) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } + } + } +} + +func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears int) error { + issueWhiteData, issueErr := models.QueryIssueWhitelist(1) + if issueErr == nil && len(issueWhiteData) > 0 { + for _, issues := range issueWhiteData { + for ; ; { + cveData, err := models.QueryIssueByPackName(issues.PackageName, beforeTime, prcnum) + if err == nil && len(cveData) > 0 { + logs.Info(cveData) + } else { + logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) + break } - // Import cve as data after 2018 - cveNumList := strings.Split(issueValue.CveNum, "-") - if cveNumList != nil && len(cveNumList) > 1 { - cveYears, yearErr := strconv.Atoi(cveNumList[1]) - if yearErr == nil { - if cveYears <= years { - icr := models.IssueCreateRecord{} - models.UpdateIssueStatus(issueValue, 4) - logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", - years, ",否则不需要在git上提交issue, cveData: ", issueValue) - sc, err := models.QueryIssueScore(issueValue.CveId) - if err != nil || sc.Id == 0 { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) - continue + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime(), ",err: ", err) + break + } + owner := beego.AppConfig.String("gitee::owner") + path := beego.AppConfig.String("gitee::path") + for index, issueValue := range cveData { + logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) + // Determine whether the issue has been processed + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝" { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) + continue + } + } + // Import cve as data after 2018 + cveNumList := strings.Split(issueValue.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if issueValue.DataSource == 1 { + years = toolYears + } else if issueValue.DataSource == 3 { + years = manYears } - GenIssueRecordData(&icr, issueValue, sc, 2) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) + if cveYears <= years { + icr := models.IssueCreateRecord{} + models.UpdateIssueStatus(issueValue, 4) + logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", + years, ",否则不需要在git上提交issue, cveData: ", issueValue) + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + continue + } + GenIssueRecordData(&icr, issueValue, sc, 2) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } + continue } - continue } } - } - // Determine whether cve has been processed - issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) - if issueExist { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) - continue - } - // Process each piece of cve data - if issueValue.Status == 0 { - err := ProcIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + // Determine whether cve has been processed + issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) + if issueExist { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) continue } - } else { - err := ProcUpdateIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + // Process each piece of cve data + if issueValue.Status == 0 { + mutex.Lock() + err := ProcIssue(issueValue, accessToken, owner, path) + mutex.Unlock() + if err != nil { + logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } else { + mutex.Lock() + err := ProcUpdateIssue(issueValue, accessToken, owner, path) + mutex.Unlock() + if err != nil { + logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } + // Collect issue record data + icr := models.IssueCreateRecord{} + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) continue } - } - // Collect issue record data - icr := models.IssueCreateRecord{} - sc, err := models.QueryIssueScore(issueValue.CveId) - if err != nil || sc.Id == 0 { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) - continue - } - GenIssueRecordData(&icr, issueValue, sc, 1) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) + GenIssueRecordData(&icr, issueValue, sc, 1) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } } } } + } else { + logs.Error(issueErr) + return issueErr } return nil } +//CreateIssue Create issue +func CreateIssue() error { + defer common.Catchs() + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return err + } + days, ok := BConfig.Int("crontab::days") + if ok != nil { + logs.Error("config crontab::days error:", ok) + return ok + } + years, ok := BConfig.Int("cve::cve_number") + if ok != nil { + logs.Error("config cve::years error:", ok) + return ok + } + manYears, ok := BConfig.Int("cve::cve_number_m") + if ok != nil { + logs.Error("config cve::manYears error:", ok) + return ok + } + toolYears, ok := BConfig.Int("cve::cve_number_t") + if ok != nil { + logs.Error("config cve::toolYears error:", ok) + return ok + } + prcnum, ok := BConfig.Int("crontab::prcnum") + if ok != nil { + logs.Error("config crontab::prcnum error:", ok) + return ok + } + issueWhitelist, ok := BConfig.Int("cve::issue_whitelist") + if ok != nil { + logs.Error("config cve::issue_whitelist error: ", ok) + return ok + } + beforeTime := common.GetBeforeTime(days) + if issueWhitelist == 1 { + limitErr := addLimitedIssue(beforeTime, prcnum, years, toolYears, manYears) + logs.Error("addLimitedIssue, err: ", limitErr) + return limitErr + } else { + unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears) + logs.Error("addUnlimitedIssue, err: ", unlimitErr) + return unlimitErr + } +} + //ProcUpdateIssue Update issue func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path string) error { // Query and modify score diff --git a/taskhandler/common.go b/taskhandler/common.go index 45809fe51d429db371f8cf912f14946efd3ca3e6..2512990a19ef0352ef0d765fada9a227bffd3a23 100644 --- a/taskhandler/common.go +++ b/taskhandler/common.go @@ -74,7 +74,7 @@ const commentCopyValue = ` %v **issue处理注意事项:** **1. 当前issue受影响的分支提交pr时, 须在pr描述中填写当前issue编号进行关联, 否则无法关闭当前issue;** -**2. 模板内容需要填写完整, 如果是非maintainer填写, 需要maintainer审核通过, 否则无法关闭当前issue;** +**2. 模板内容需要填写完整, 无论是受影响或者不受影响都需要填写完整内容, 否则无法关闭当前issue;** **3. 以下为模板中需要填写完整的内容, 请复制到评论区回复, 注: 内容的标题名称(影响性分析说明, openEuler评分, 受影响版本排查(受影响/不受影响))不能省略,省略后cve-manager将无法正常解析填写内容.** ************************************************************************ 影响性分析说明: diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index 5ad92c9156a0508af03526ce2caca5cd33abbe6f..dec46635a67835d03aa49b488666adc56685ad76 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -12,8 +12,11 @@ import ( "github.com/astaxie/beego/logs" "strconv" "strings" + "sync" ) +var updateLock sync.Mutex + func CreateIssueData(issueTemp *models.IssueTemplate, cve models.VulnCenter, sc models.Score, resp map[string]interface{}, path, assignee, issueType, labels, owner string) *models.IssueTemplate { issueTemp.CveId = cve.CveId @@ -92,6 +95,12 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st var it models.IssueTemplate it.CveId = cve.CveId _, err := models.GetIssueTemplet(&it) + sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} + specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + it.Assignee = sia.Assignee + assignee = sia.Assignee + } if err && it.IssueNum != "" && len(it.IssueNum) > 0 { if it.Assignee == "" || len(it.Assignee) == 0 { it.Assignee = assignee @@ -265,6 +274,11 @@ func UpdateIssueToGit(accessToken string, owner string, path string, cve models.VulnCenter, its models.IssueTemplate) (string, error) { logs.Info("更新模板请求参数: cve: ", cve, ",its: ", its, ", owner: ", owner, ",path: ", path) //labels := its.IssueLabel + sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} + specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + its.Assignee = sia.Assignee + } labels := "" if its.IssueLabel != "" && len(its.IssueLabel) > 1 { labels = its.IssueLabel @@ -292,7 +306,9 @@ func UpdateIssueToGit(accessToken string, owner string, path string, cve, sc, OpenEulerScore, score, labels, its, 3, its.IssueType, pkgLink, brandArray) logs.Info("isssue_body: ", requestBody) if requestBody != "" && len(requestBody) > 1 { + updateLock.Lock() resp, err := util.HTTPPatch(url, requestBody) + updateLock.Unlock() if err != nil { logs.Error("更新issue失败, cveNum: ", cve.CveNum, "err: ", err) return "", err diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 38ffb1c825736ca96379cc866186e76c06b956ed..390aec82f4e10b4c46ac2300704912395d1cc6b2 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -673,6 +673,7 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum return true, nil } +// Synchronize the data returned by the Chinese Academy of Sciences func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) (bool, error) { if cveData.Ids == "" || cveData.CveNum == "" { logs.Error("当前数据cveNum 为空暂不处理,data: ", cveData) @@ -684,7 +685,7 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) logs.Error("config init error:", err) return false, err } - years, confOk := BConfig.Int("cve::cve_number") + years, confOk := BConfig.Int("cve::cve_number_t") if confOk != nil { logs.Error("config cve::years error:", confOk) return false, errors.New("数据错误，暂时不处理") @@ -1694,7 +1695,7 @@ func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { du := "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" //du := beego.AppConfig.String("excel::v_pack_20_03_url") du = beego.AppConfig.DefaultString("excel::v_pack_20_03_url", du) - localPath := filepath.Join(dir, "release-package.CSV") + localPath := filepath.Join(dir, affectBranch + "_release-package.CSV") err = downloadPackageFile(localPath, du) if err != nil { logs.Error(err) diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 30f10c741fe28dd087d11b125a699095fcd5b797..8fb1bbacb12fb76ed9f9cafa02ee38ca8aea18be 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -775,7 +775,6 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I if ok != nil { saTimeStampZone = 3600 * 8 } - //owner := "src-openeuler" st := util.TimeStrToInt(startTime, "2006-01-02") chData := make([]IssueAndPkg, 0) for _, v := range pkgList { diff --git a/util/http.go b/util/http.go index 70007aadfe4a68d985869675054de96d49d00973..345c28807e387cd638d6741a3a3b3d56dca3960b 100644 --- a/util/http.go +++ b/util/http.go @@ -2,6 +2,7 @@ package util import ( "bytes" + "cvevulner/common" "encoding/json" "fmt" "github.com/astaxie/beego/logs" @@ -22,6 +23,7 @@ type RequestInfo struct { //HTTPPatch patch request func HTTPPatch(url string, requestBody string) (map[string]interface{}, error) { req, err := http.NewRequest("PATCH", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() if err != nil { return nil, err } @@ -60,6 +62,7 @@ func HTTPPatch(url string, requestBody string) (map[string]interface{}, error) { //HTTPPost post request func HTTPPost(url string, requestBody string) (map[string]interface{}, error) { req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() if err != nil { return nil, err } @@ -98,6 +101,7 @@ func HTTPPost(url string, requestBody string) (map[string]interface{}, error) { //HTTPPost1 post request func HTTPPost1(url string, requestBody string) ([]map[string]interface{}, error) { req, err := http.NewRequest("POST", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() if err != nil { return nil, err }