diff --git a/conf/app.conf b/conf/app.conf index 6e6d363b6b6bfe96f23616738c5bd4e023713e2e..4a001ab0e6d294c334cebea12ef8980bcfdcf629 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -48,12 +48,12 @@ maxsize=204800 ymalflag = 2 getymal = 0 0 1 * * 1 cveflag = 2 -getcve = 0 17 11 * * * +getcve = 0 34 11 * * * oricveflag = 2 oricvecheck = 0 16 19 * * * getissueflag = 2 -getissue = 0 43 14 * * * -issueflag = 1 +getissue = 0 20 10 * * * +issueflag = 2 createissue = 0 38 15 * * * test = 0/10 * * * * * gittokenflag = 2 @@ -68,6 +68,8 @@ unlockflag = 2 unlock = 0 30 19 * * * hookflag = 2 prochook = 0 34 17 * * * +exceptflag = 1 +exceptcve = 0 3 19 * * * [gitee] @@ -118,6 +120,9 @@ issue_whitelist = 2 affected_branchs = openEuler-20.03-LTS # Close the highest privilege of issue close_issue_privilege = 2 +# abnormal cve status ,Use "," to separate multiple states +abn_cve_status =3 + [reflink] comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual.md @@ -127,7 +132,9 @@ forcerewrite = false snprefix = op-2020-10- snsuffix = 1002 # Version package excel download address -v_pack_20_03_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv +# example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; +# openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv +v_pack_20_03_url = "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" # Time difference in different time zones sa_timestamp_zone = 28800 diff --git a/conf/product_app.conf b/conf/product_app.conf index c392d320b6c8cb6867f97b29bb7aa8dcd9a4afb2..12eb12e94030db88f835be40e7852207dd2cf615 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -61,7 +61,7 @@ gittokenflag = 2 issueoath = * * */20 * * * genexcelflag = 1 genexcel = 0 30 7 * * * -days = -100 +days = -50 prcnum = 50 printlogflag = 1 printlog = 0 0 1 * * 1 @@ -128,7 +128,9 @@ forcerewrite = false snprefix = op-2020-10- snsuffix = 1002 # Version package excel download address -v_pack_20_03_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv +# example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; +# openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv +v_pack_20_03_url = "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" # Time difference in different time zones sa_timestamp_zone = 28800 @@ -136,6 +138,8 @@ sa_timestamp_zone = 28800 updateinfo_path = download/updateinfo.xml [email] + + email_name = "${EMAIL_NAME||***}" email_pwd = "${EMAIL_PWD||***}" email_host = smtp.gmail.com diff --git a/controllers/file.go b/controllers/file.go index 57fc1880ea437135f5cb486f048012a706b4f28a..9437b8f9518f690d0dfc0ae2ad26a1ebb793b06a 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -10,7 +10,9 @@ import ( "github.com/astaxie/beego/logs" "path/filepath" "regexp" + "strconv" "strings" + "sync" "time" ) @@ -104,28 +106,118 @@ func (f *FileController) TriggerCveData() { f.Ctx.WriteString(`Error: please enter the correct start time in a format like this "yyyy-MM-dd".`) return } - affectBranchsxList := []string{} - affectedBranchs := beego.AppConfig.String("cve::affected_branchs") - if affectedBranchs != "" && len(affectedBranchs) > 0 { - affectBranchsxList = strings.Split(affectedBranchs, ",") - } else { - affectBranchsxList = append(affectBranchsxList, "openEuler-20.03-LTS") - } - for _, affectBranch := range affectBranchsxList { - now := time.Now().Unix() - en := fmt.Sprintf("cve与安全公告%v_%v.xlsx", affectBranch, now) - fileCode := common.EncryptMd5(en) - //call the generate excel task by new thread - er = models.ExportRecord{FileName: en, FileCode: fileCode, State: 0, CreateTime: timeUnix} - err = er.Insert() - if err != nil { - f.Ctx.WriteString(fmt.Sprintf("error:%v", err)) - } else { - //return the success notice - go taskhandler.GenerateExcelTrigger(en, startTime, fileCode, affectBranch) - f.Ctx.WriteString(fmt.Sprintf("Success：The name of the excel file generated this time is: %s. "+ - "It takes some time to generate the excel file. "+ - "You can try to call the download file interface and pass in the param fileCode=%s to be downloaded.", en, fileCode)) + dir := beego.AppConfig.DefaultString("fileDir", "download") + dirErr := util.MakeDir(dir) + if dirErr != nil { + logs.Error(dirErr) + f.Ctx.WriteString(`Error: File execution permission error, continue processing later.`) + return + } + var wgCrvf sync.WaitGroup + componentMap := make(map[string]taskhandler.ComponentInfo) + cvrfFileList := make(map[string][]string) + cvrfFileMap := make(map[string]taskhandler.CvrfSa) + du := "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" + //du := beego.AppConfig.String("excel::v_pack_20_03_url") + du = beego.AppConfig.DefaultString("excel::v_pack_20_03_url", du) + csvPathList := strings.Split(du, ";") + if len(csvPathList) > 0 { + for _, csvP := range csvPathList { + openBranchx := strings.Split(csvP, "@") + if len(openBranchx) == 2 { + affectBranch := openBranchx[0] + csvDownPath := openBranchx[1] + now := time.Now().Unix() + en := fmt.Sprintf("cve与安全公告%v_%v.xlsx", openBranchx[0], now) + fileCode := common.EncryptMd5(en) + //call the generate excel task by new thread + er = models.ExportRecord{FileName: en, FileCode: fileCode, State: 0, CreateTime: timeUnix} + err = er.Insert() + if err != nil { + f.Ctx.WriteString(fmt.Sprintf("error:%v", err)) + } else { + // return the success notice + // Need to be executed sequentially + wgCrvf.Add(1) + go taskhandler.GenerateExcelTrigger(&wgCrvf, en, startTime, fileCode, affectBranch, + csvDownPath, dir, cvrfFileList, componentMap, cvrfFileMap) + // Return the result first, continue processing the data + f.Ctx.WriteString(fmt.Sprintf("Success：The name of the excel file generated this time is: %s. "+ + "It takes some time to generate the excel file. "+ + "You can try to call the download file interface and pass in the param fileCode=%s to be downloaded.", en, fileCode)) + } + } + } + } + wgCrvf.Wait() + // Generate cvrf file + GenAndUploadCvrf(cvrfFileMap, cvrfFileList) +} + +// generate and upload cvrf.xml +func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, cvrfFileList map[string][]string) { + if len(cvrfFileMap) > 0 { + for cvrfKey, cvrfVule := range cvrfFileMap { + taskhandler.WriteCvrfXml(cvrfKey, &cvrfVule) + } + } + totalFileSlice := make([]string, 0) + // File name and data stored in database + fileSlice, fOk := cvrfFileList[taskhandler.CVRFFKEY] + if fOk { + subFileSlice := make([]string, 0) + for _, fPath := range fileSlice { + _, fileName := filepath.Split(fPath) + // File storage to db + recordErr := taskhandler.RecordCrvfInfo(fPath, fileName, taskhandler.FIXEDFLAGE) + logs.Info("recordErr: ", recordErr) + // Upload file, pending + + subFileSlice = append(subFileSlice, fileName) + totalFileSlice = append(totalFileSlice, fPath) + // Upload successfully, modify file status + + } + //if len(subFileSlice) > 0 { + // taskhandler.SendCvrfEmail(subFileSlice) + //} + ////Delete local files + //taskhandler.DelFile(fileSlice) + } + unaffFileSlice, unffOk := cvrfFileList[taskhandler.UNAFFECTCVRFKEY] + if unffOk { + subFileSlice := make([]string, 0) + for _, fPath := range unaffFileSlice { + _, fileName := filepath.Split(fPath) + // File storage to db + recordErr := taskhandler.RecordCrvfInfo(fPath, fileName, taskhandler.UNAFFECTFLAG) + logs.Info("recordErr: ", recordErr) + // Upload file, pending + + subFileSlice = append(subFileSlice, fileName) + totalFileSlice = append(totalFileSlice, fPath) + // Upload successfully, modify file status + + } + } + if len(totalFileSlice) > 0 { + //taskhandler.SendCvrfEmail(totalFileSlice) + dir := "download" + zipFileName := "cvrf-" + common.GetCurDate() + "_" + strconv.Itoa(time.Now().Nanosecond()) + ".zip" + zipFileName = filepath.Join(dir, zipFileName) + zipErr := taskhandler.ZipFiles(zipFileName, totalFileSlice, dir, dir) + if zipErr != nil { + logs.Error("File compression failed: err: ", zipErr) + } + // send email + sendError := taskhandler.SendEmail(zipFileName, 1) + if sendError != nil { + logs.Error("SendEmail, sendErr: ", sendError) + return } + totalFileSlice = append(totalFileSlice, zipFileName) } + // Delete local files + taskhandler.DelFile(totalFileSlice) + logs.Info(cvrfFileList[taskhandler.BRANCHSKEY], ",End of generating cvrf format file this time") } diff --git a/cve-py/emailtask/sendemail.py b/cve-py/emailtask/sendemail.py index 9f6c969c6ef95467913bb87d7491b92dfeae1897..e51b3f08f122787f19968c348055d6ae10ea71e1 100644 --- a/cve-py/emailtask/sendemail.py +++ b/cve-py/emailtask/sendemail.py @@ -1,5 +1,5 @@ #!/usr/bin/python -# -*- coding: UTF-8 -*- +# -*- coding: utf-8 -*- ################################################################################ # @@ -18,7 +18,7 @@ import os import email.mime.multipart import email.mime.text import email.mime.application as application - +import email.utils def send_email(smtp_host, smtp_port, send_addr, password, recipient_addrs, path, subject='', content=''): """ @@ -34,7 +34,7 @@ def send_email(smtp_host, smtp_port, send_addr, password, recipient_addrs, path, :return:None """ msg = email.mime.multipart.MIMEMultipart() - msg['from'] = send_addr + msg['from'] = email.utils.formataddr(["cve-manager", send_addr]) msg['to'] = recipient_addrs msg['subject'] = subject content = content diff --git a/cve-py/main.py b/cve-py/main.py index f988f87b7beae364ba70a7f312d389af54cf89b5..73fa067632a22d7d5ae5a72acd7e24c8c66c0664 100644 --- a/cve-py/main.py +++ b/cve-py/main.py @@ -17,6 +17,8 @@ Date: 10/22/2020 11:01 AM """ from controller import timertaskcontroller + + if __name__ == '__main__': print("The program starts, waiting for the timing task to execute") timertaskcontroller.timertask() diff --git a/doc/image/Framework.png b/doc/image/Framework.png index 6b40d66c3d1cfcf9cabde6cf2dadda3142212018..eacb23ff606557a026aff12b560ef60dd69718c0 100644 Binary files a/doc/image/Framework.png and b/doc/image/Framework.png differ diff --git a/doc/sql/db_struct.sql b/doc/sql/db_struct.sql index 42894f7e6fd1ce8f4f33bddb46464979c5980325..61d63d1d4cbc3c473d1a51e4cf2d5bfcf8eae1c8 100644 --- a/doc/sql/db_struct.sql +++ b/doc/sql/db_struct.sql @@ -28,6 +28,30 @@ CREATE TABLE `cve_admin_user` ( PRIMARY KEY (`user_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8; +/*Table structure for table `cve_cvrf_sa_record` */ + +DROP TABLE IF EXISTS `cve_cvrf_sa_record`; + +CREATE TABLE `cve_cvrf_sa_record` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `openeuler_sa_num` varchar(128) NOT NULL DEFAULT '', + `xml_content` text CHARACTER SET utf8 COLLATE utf8_general_ci NOT NULL, + `cur_md5` varchar(128) NOT NULL DEFAULT '', + `update_md5` varchar(128) NOT NULL DEFAULT '', + `cve_status` tinyint NOT NULL DEFAULT '1', + `is_export` tinyint DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + `pack_name` varchar(512) NOT NULL DEFAULT '', + `cve_num` varchar(1024) NOT NULL DEFAULT '', + `affect_flag` tinyint NOT NULL DEFAULT '1', + PRIMARY KEY (`id`), + UNIQUE KEY `openeuler_sa_num` (`openeuler_sa_num`), + UNIQUE KEY `cve_cvrf_md5` (`cur_md5`), + KEY `cve_cvrf_sa_record_cve_num` (`cve_num`) +) ENGINE=InnoDB AUTO_INCREMENT=15 DEFAULT CHARSET=utf8; + /*Table structure for table `cve_email_list` */ DROP TABLE IF EXISTS `cve_email_list`; @@ -38,7 +62,7 @@ CREATE TABLE `cve_email_list` ( `email_type` tinyint NOT NULL DEFAULT '1', PRIMARY KEY (`id`), KEY `cve_email_list_email_type` (`email_type`) -) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8; /*Table structure for table `cve_export_record` */ @@ -52,7 +76,7 @@ CREATE TABLE `cve_export_record` ( `state` tinyint NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `file_name` (`file_name`) -) ENGINE=InnoDB AUTO_INCREMENT=112 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=162 DEFAULT CHARSET=utf8; /*Table structure for table `cve_file_hash` */ @@ -63,7 +87,7 @@ CREATE TABLE `cve_file_hash` ( `file_name` varchar(50) DEFAULT NULL, `file_hash` varchar(50) DEFAULT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=34 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=141 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_open_euler` */ @@ -164,7 +188,7 @@ CREATE TABLE `cve_git_repo_groups` ( `group_name` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY (`group_id`), KEY `cve_git_repo_groups_group_name` (`group_name`) -) ENGINE=InnoDB AUTO_INCREMENT=147 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=151 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_sub_pack` */ @@ -252,7 +276,7 @@ CREATE TABLE `cve_gite_origin_issue` ( PRIMARY KEY (`id`), UNIQUE KEY `issue_id` (`issue_id`), UNIQUE KEY `number` (`number`,`cve_number`,`repo_path`) -) ENGINE=InnoDB AUTO_INCREMENT=5804 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=9749 DEFAULT CHARSET=utf8; /*Table structure for table `cve_gite_repo` */ @@ -273,7 +297,7 @@ CREATE TABLE `cve_gite_repo` ( `delete_time` varchar(255) DEFAULT NULL, PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_group_id` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=35901 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=108916 DEFAULT CHARSET=utf8; /*Table structure for table `cve_gite_repo_branch` */ @@ -304,7 +328,7 @@ CREATE TABLE `cve_gite_repo_member` ( PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_member_group_id` (`group_id`), CONSTRAINT `member_groups_group_id` FOREIGN KEY (`group_id`) REFERENCES `cve_git_repo_groups` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2216 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=6766 DEFAULT CHARSET=utf8; /*Table structure for table `cve_ip_white` */ @@ -352,7 +376,7 @@ CREATE TABLE `cve_issue_create_record` ( PRIMARY KEY (`id`), KEY `cve_issue_create_record_cve_id` (`cve_id`), KEY `cve_issue_create_record_cve_num` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=1469 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=1477 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_hooks` */ @@ -380,7 +404,7 @@ CREATE TABLE `cve_issue_hooks` ( UNIQUE KEY `cve_issue_hooks_owner_repo_status_un` (`owner`,`repo`,`hook_url`,`status`), KEY `cve_issue_hooks_Cve_id` (`cve_id`), KEY `cve_issue_hooks_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=29 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_repo_whitelist` */ @@ -398,7 +422,7 @@ CREATE TABLE `cve_issue_repo_whitelist` ( PRIMARY KEY (`repo_id`), KEY `cve_issue_repo_whitelist_package_name` (`package_name`), KEY `cve_issue_repo_whitelist_version` (`version`) -) ENGINE=InnoDB AUTO_INCREMENT=143 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=338 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_template` */ @@ -448,7 +472,7 @@ CREATE TABLE `cve_issue_template` ( UNIQUE KEY `cve_issue_template_num_compone_versio` (`cve_num`,`owned_component`,`owned_version`), KEY `cve_issue_template_cve_id` (`cve_id`), KEY `cve_issue_template_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=2216 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2389 DEFAULT CHARSET=utf8; /*Table structure for table `cve_open_euler_s_a` */ @@ -462,7 +486,7 @@ CREATE TABLE `cve_open_euler_s_a` ( PRIMARY KEY (`openeuler_id`), UNIQUE KEY `openeuler_sa_num` (`openeuler_sa_num`), KEY `cve_open_euler_s_a_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2784 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2875 DEFAULT CHARSET=utf8; /*Table structure for table `cve_open_guss_yaml` */ @@ -516,7 +540,7 @@ CREATE TABLE `cve_origin_excel` ( `is_export` tinyint DEFAULT '1', PRIMARY KEY (`cve_id`) USING BTREE, UNIQUE KEY `cve_origin_excel_num_un` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=364 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; +) ENGINE=InnoDB AUTO_INCREMENT=380 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; /*Table structure for table `cve_origin_upstream` */ @@ -549,7 +573,7 @@ CREATE TABLE `cve_origin_upstream` ( KEY `cve_origin_upstream_cve_packname` (`cve_packname`), KEY `cve_origin_upstream_git_packname` (`git_packname`), KEY `cve_origin_upstream_version` (`version`) -) ENGINE=InnoDB AUTO_INCREMENT=1847 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=1857 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config` */ @@ -561,7 +585,7 @@ CREATE TABLE `cve_origin_upstream_config` ( `nodes` varchar(32) DEFAULT NULL, PRIMARY KEY (`conf_id`), KEY `cve_origin_upstream_config_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node` */ @@ -573,7 +597,7 @@ CREATE TABLE `cve_origin_upstream_config_node` ( `operator` varchar(256) DEFAULT NULL, PRIMARY KEY (`node_id`), KEY `cve_origin_upstream_config_node_conf_id` (`conf_id`) -) ENGINE=InnoDB AUTO_INCREMENT=24801 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=24811 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node_cpe` */ @@ -587,7 +611,7 @@ CREATE TABLE `cve_origin_upstream_config_node_cpe` ( `vulner_able` varchar(64) DEFAULT NULL, PRIMARY KEY (`cpe_id`), KEY `cve_origin_upstream_config_node_cpe_node_id` (`node_id`) -) ENGINE=InnoDB AUTO_INCREMENT=767810 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=767824 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_desc` */ @@ -600,7 +624,7 @@ CREATE TABLE `cve_origin_upstream_desc` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`desc_id`), KEY `cve_origin_upstream_desc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_event` */ @@ -615,7 +639,7 @@ CREATE TABLE `cve_origin_upstream_event` ( `description` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`event_id`), KEY `cve_origin_upstream_event_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest` */ @@ -627,7 +651,7 @@ CREATE TABLE `cve_origin_upstream_fix_suggest` ( `detail` varchar(1024) DEFAULT NULL, PRIMARY KEY (`fix_id`), KEY `cve_origin_upstream_fix_suggest_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13125 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13135 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest_ref` */ @@ -665,7 +689,7 @@ CREATE TABLE `cve_origin_upstream_impact` ( `impact` varchar(32) DEFAULT NULL, PRIMARY KEY (`impact_id`), KEY `cve_origin_upstream_impact_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score` */ @@ -681,7 +705,7 @@ CREATE TABLE `cve_origin_upstream_impact_score` ( `score_status` tinyint DEFAULT '1', PRIMARY KEY (`score_id`), KEY `cve_origin_upstream_impact_score_impact_id` (`impact_id`) -) ENGINE=InnoDB AUTO_INCREMENT=26355 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=26375 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v2` */ @@ -710,7 +734,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v2` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v2_id`), KEY `cve_origin_upstream_impact_score_v2_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v3` */ @@ -736,7 +760,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v3` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v3_id`), KEY `cve_origin_upstream_impact_score_v3_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_poc` */ @@ -753,7 +777,7 @@ CREATE TABLE `cve_origin_upstream_poc` ( `desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`poc_id`), KEY `cve_origin_upstream_poc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13178 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_reference` */ @@ -768,7 +792,7 @@ CREATE TABLE `cve_origin_upstream_reference` ( `tags` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`ref_id`), KEY `cve_origin_upstream_reference_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=260700 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=260775 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_vul_type` */ @@ -782,7 +806,7 @@ CREATE TABLE `cve_origin_upstream_vul_type` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`vul_id`), KEY `cve_origin_upstream_vul_type_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13125 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=13135 DEFAULT CHARSET=utf8; /*Table structure for table `cve_other_user` */ @@ -811,7 +835,7 @@ CREATE TABLE `cve_package` ( `pack_url` varchar(2048) DEFAULT NULL, PRIMARY KEY (`id`), KEY `cve_package_sec_id` (`sec_id`) -) ENGINE=InnoDB AUTO_INCREMENT=18606 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=53784 DEFAULT CHARSET=utf8; /*Table structure for table `cve_package_cpe` */ @@ -824,7 +848,7 @@ CREATE TABLE `cve_package_cpe` ( `create_time` varchar(32) DEFAULT NULL, PRIMARY KEY (`id`), KEY `cve_package_cpe_packname` (`packname`) -) ENGINE=InnoDB AUTO_INCREMENT=2472 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2480 DEFAULT CHARSET=utf8; /*Table structure for table `cve_score` */ @@ -870,7 +894,7 @@ CREATE TABLE `cve_score` ( PRIMARY KEY (`id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_score_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2784 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2875 DEFAULT CHARSET=utf8; /*Table structure for table `cve_score_record` */ @@ -885,7 +909,7 @@ CREATE TABLE `cve_score_record` ( `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `cve_score_record_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=3558 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3659 DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_notice` */ @@ -911,7 +935,7 @@ CREATE TABLE `cve_security_notice` ( PRIMARY KEY (`sec_id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_security_notice_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2784 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2875 DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_reviewer` */ @@ -942,7 +966,7 @@ CREATE TABLE `cve_spec_error` ( PRIMARY KEY (`id`), UNIQUE KEY `cve_spec_error_cve_num` (`cve_num`), KEY `cve_spec_error_cve_owner` (`cve_owner`) -) ENGINE=InnoDB AUTO_INCREMENT=471 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=8226 DEFAULT CHARSET=utf8; /*Table structure for table `cve_spec_issue_assigness` */ @@ -983,7 +1007,7 @@ CREATE TABLE `cve_vuln_center` ( PRIMARY KEY (`cve_id`), UNIQUE KEY `cve_vuln_center_num_pack_v_un` (`cve_num`,`cve_version`,`pack_name`), KEY `cve_vuln_center_cve_num` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=2790 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2881 DEFAULT CHARSET=utf8; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; diff --git a/go.mod b/go.mod index 4205ccc91b4e562834332a9142805032bbc624f4..20e1f359bce26e9884f49669439871165266fa89 100644 --- a/go.mod +++ b/go.mod @@ -29,6 +29,7 @@ require ( golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff // indirect golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7 // indirect golang.org/x/text v0.3.4 // indirect + gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.3.0 // indirect ) diff --git a/models/excel.go b/models/excel.go index 807f4e9a65496f6cb0442f7ad2aca4560675a794..9173342ce15f0efee705d116e4464355185f6b45 100644 --- a/models/excel.go +++ b/models/excel.go @@ -71,3 +71,21 @@ func (elt *EmailList) Read(field ...string) ([]EmailList, error) { logs.Error("查询 cve_email_list ,err: ", err) return el, err } + +func GetCvrfRecord(cfr *CvrfSaRecord, colName ...string) error { + o := orm.NewOrm() + err := o.Read(cfr, colName...) + return err +} + +func InsertCvrfRecord(cfr *CvrfSaRecord) (int64, error){ + o := orm.NewOrm() + num, err := o.Insert(cfr) + return num, err +} + +func UpdateCvrfRecord(cfr *CvrfSaRecord, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(cfr, fields...) + return err +} \ No newline at end of file diff --git a/models/giteeissue.go b/models/giteeissue.go index 25d72833da954245571bbd6daba09d30aff31d5c..39a0d0d84356e06dfb0e4a20f56bd1ea705f819d 100644 --- a/models/giteeissue.go +++ b/models/giteeissue.go @@ -181,7 +181,7 @@ func parseOldTplToLoopHole(lp *Loophole, body string) { if len(sm) > 0 && len(sm[0]) > 1 { for _, v := range sm[0][1:] { if v != "" { - lp.Components = strings.ToLower(util.TrimString(v)) + lp.Components = util.TrimString(v) break } } @@ -269,7 +269,7 @@ func parseNewTplToLoopHole(lp *Loophole, body string) { if len(sm) > 0 && len(sm[0]) > 1 { for _, v := range sm[0][1:] { if v != "" { - lp.Components = strings.ToLower(util.TrimString(v)) + lp.Components = util.TrimString(v) break } } diff --git a/models/modeldb.go b/models/modeldb.go index 3d69d40756da941927f8ada5866775519cf942fa..f07503e071d6933c899214a7c5012d7ec6a45242 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -641,11 +641,14 @@ type SpecError struct { type CvrfSaRecord struct { Id int64 `orm:"pk;auto;column(id)"` OpenEulerSANum string `orm:"size(128);column(openeuler_sa_num);unique" description:"安全公告"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + CveNum string `orm:"size(1024);column(cve_num);index" description:"cve编号"` XmlContent string `orm:"text;column(xml_content)" description:"xml内容"` Md5 string `orm:"size(128);column(cur_md5)" description:"当前内容生成的MD5"` UpdateMd5 string `orm:"size(128);column(update_md5)" description:"内容更新后生成的MD5"` Status int8 `orm:"default(1);column(cve_status)" description:"1:初始化; 2: 已更新; 3: 已删除"` IsExport int8 `orm:"default(1);column(is_export);null" description:"1: 未导出; 2: 已导出"` + AffectFlag int8 `orm:"default(1);column(affect_flag)" description:"1:受影响标识; 2: 不受影响标识; 3: 其他"` CreateTime string `orm:"size(32);column(create_time)"` UpdateTime string `orm:"size(32);column(update_time);null"` DeleteTime string `orm:"size(32);column(delete_time);null"` @@ -679,7 +682,7 @@ func CreateDb() bool { new(GiteRepo), new(GiteRepoMember), new(GiteRepoBranch), new(PackageCpe), new(EmailList), new(IssueAssignee), new(IssueRepoWhitelist), new(IssueCreateRecord), new(OpenGussYaml), - new(SpecIssueAssigness), new(SpecError), + new(SpecIssueAssigness), new(SpecError), new(CvrfSaRecord), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/routers/commentsRouter____cvevulner_controllers.go b/routers/commentsRouter____cvevulner_controllers.go new file mode 100644 index 0000000000000000000000000000000000000000..e2a939601e78e632e68ce78d36da56c67f4f5348 --- /dev/null +++ b/routers/commentsRouter____cvevulner_controllers.go @@ -0,0 +1,226 @@ +package routers + +import ( + "github.com/astaxie/beego" + "github.com/astaxie/beego/context/param" +) + +func init() { + + beego.GlobalControllerRouter["cvevulner/controllers:CveDetailController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:CveDetailController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:CveErrorFeedBackController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:CveErrorFeedBackController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:CveIssueWhiteListController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:CveIssueWhiteListController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:FileController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:FileController"], + beego.ControllerComments{ + Method: "DownLoadExcelByFileCode", + Router: "/downloadExcel", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:FileController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:FileController"], + beego.ControllerComments{ + Method: "DownloadLastExcel", + Router: "/lastExcel", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:FileController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:FileController"], + beego.ControllerComments{ + Method: "TriggerCveData", + Router: "/triggerCveData", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:HookEventControllers"] = append(beego.GlobalControllerRouter["cvevulner/controllers:HookEventControllers"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:IssueOathCallbackController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:IssueOathCallbackController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "GetAll", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Get", + Router: "/:objectId", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Put", + Router: "/:objectId", + AllowHTTPMethods: []string{"put"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:ObjectController"], + beego.ControllerComments{ + Method: "Delete", + Router: "/:objectId", + AllowHTTPMethods: []string{"delete"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:PackagesController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:PackagesController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:PackagesInfoController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:PackagesInfoController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "GetAll", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Get", + Router: "/:uid", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Put", + Router: "/:uid", + AllowHTTPMethods: []string{"put"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Delete", + Router: "/:uid", + AllowHTTPMethods: []string{"delete"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Login", + Router: "/login", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserController"], + beego.ControllerComments{ + Method: "Logout", + Router: "/logout", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserLoginController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserLoginController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + + beego.GlobalControllerRouter["cvevulner/controllers:UserUploadController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:UserUploadController"], + beego.ControllerComments{ + Method: "Post", + Router: "/", + AllowHTTPMethods: []string{"post"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + +} diff --git a/taskhandler/common.go b/taskhandler/common.go index 92b9549494ed1778bbcbb317d29f3c1b4820bdf3..09a8680ad909c166bd680039699be980e38e2754 100644 --- a/taskhandler/common.go +++ b/taskhandler/common.go @@ -3,10 +3,12 @@ package taskhandler import ( "cvevulner/common" "cvevulner/models" + "encoding/base64" "fmt" "github.com/astaxie/beego" "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" + "os" "reflect" "strconv" "strings" @@ -624,3 +626,19 @@ func openEulerScoreProc(openEulerScore float64) (CveLevel string){ } return CveLevel } + +func PathExists(path string) (bool, error) { + _, err := os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +func baseStdEncode(srcBtye []byte) string { + encoding := base64.StdEncoding.EncodeToString(srcBtye) + return encoding +} \ No newline at end of file diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 025976e1d0b6e222d9393573c872d5f31af15b11..e3a81caf893bf45b8714c2f4809d700dde0c4911 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -54,7 +54,7 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNu OpenNumList := strings.Split(osx.OpenEulerSANum, "-") OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) if err == nil { - OpenNum += 10 + OpenNum += 1 } else { OpenNum = openeulerNum } @@ -282,7 +282,7 @@ func InsertCveExcelGroups(cveData models.OriginExcel, cveRef string, openeulerNu OpenNumList := strings.Split(osx.OpenEulerSANum, "-") OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) if err == nil { - OpenNum += 10 + OpenNum += 1 } else { OpenNum = openeulerNum } @@ -329,7 +329,7 @@ func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum OpenNumList := strings.Split(osx.OpenEulerSANum, "-") OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) if err == nil { - OpenNum += 10 + OpenNum += 1 } else { OpenNum = openeulerNum } @@ -650,7 +650,7 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum OpenNumList := strings.Split(osx.OpenEulerSANum, "-") OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) if err == nil { - OpenNum += 10 + OpenNum += 1 } else { OpenNum = openeulerNum } @@ -975,10 +975,10 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c vul.CveLevel = openEulerScoreProc(v2) var sec models.SecurityNotice sec.CveNum = cveData.CveNumber - sec.InfluenceComponent = lop.Components + sec.InfluenceComponent = cveData.RepoPath sec.Status = 0 sec.AffectStatus = "Fixed" - sec.Summary = lop.Components + " security update" + sec.Summary = cveData.RepoPath + " security update" sec.Description = RemoveSubstring(goe.Decription, specCharList) + "\n\n" + "Security Fix(es):" + "\n\n" + RemoveSubstring(lop.BriefIntroduction, specCharList) if sec.Description != "" && len(sec.Description) > 1 { @@ -1000,7 +1000,7 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c } else { affectBrands = lop.InfProduct } - sec.Introduction = "An update for " + lop.Components + " is now available for " + affectBrands + "." + sec.Introduction = "An update for " + cveData.RepoPath + " is now available for " + affectBrands + "." sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openEuler Security has rated this" + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + " Scoring System(CVSS)base score,which gives a detailed severity rating," + @@ -1103,7 +1103,7 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c logs.Info("no issueTemp: ", err) } issueTemp.CveNum = cveData.CveNumber - issueTemp.OwnedComponent = lop.Components + issueTemp.OwnedComponent = cveData.RepoPath issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) issueTemp.NVDScore = nVDScore issueTemp.OpenEulerScore = openEulerScore @@ -1406,7 +1406,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c OpenNumList := strings.Split(osx.OpenEulerSANum, "-") OpenNum, err := strconv.Atoi(OpenNumList[len(OpenNumList)-1]) if err == nil { - OpenNum += 10 + OpenNum += 1 } else { OpenNum = openeulernum } @@ -1446,7 +1446,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c templateErr := models.GetIssueTemplateByColName(&issueTemp, "cve_id") if templateErr == nil && issueTemp.TemplateId > 0 { //issueTemp.CveNum = cveData.CveNumber - //issueTemp.OwnedComponent = lop.Components + issueTemp.OwnedComponent = cveData.RepoPath //issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) if openError == nil && openEulerScore > 0 && issueTemp.OpenEulerScore == 0 { @@ -1509,7 +1509,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c issueTemp.CveLevel = vul.CveLevel } else { issueTemp.CveNum = cveData.CveNumber - issueTemp.OwnedComponent = lop.Components + issueTemp.OwnedComponent = cveData.RepoPath issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) issueTemp.NVDScore = nVDScore openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) @@ -1864,54 +1864,45 @@ func GenerateExcelTask1() error { } //GenerateExcelTrigger generate cve&security notice excel file by pr merge and influence package release. -func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { - //FilterCveExported() +func GenerateExcelTrigger(wgx *sync.WaitGroup, fileName, startTime, fileCode, affectBranch, + csvDownPath, dir string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { + defer wgx.Done() logs.Info(affectBranch, ", GenerateExcelTrigger start....") - dir := beego.AppConfig.DefaultString("fileDir", "download") - err := util.MakeDir(dir) + filePath := beego.AppConfig.String("xml::updateinfo_path") + fr := models.ExportRecord{FileName: fileName} + err := fr.Read("file_name") + if err != nil { + logs.Error("sddds", err) + return + } + fileName = filepath.Join(dir, fileName) + localPath := filepath.Join(dir, affectBranch+"_release-package.CSV") + err = downloadPackageFile(localPath, csvDownPath) if err != nil { logs.Error(err) + fr.State = 2 + _ = fr.Update("state") + return } - //fileDir := beego.AppConfig.DefaultString("fileDir", "download") - filePath := beego.AppConfig.String("xml::updateinfo_path") - if affectBranch == "openEuler-20.03-LTS" { - fr := models.ExportRecord{FileName: fileName} - err = fr.Read("file_name") - if err != nil { - logs.Error("sddds", err) - return - } - fileName = filepath.Join(dir, fileName) - du := "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" - //du := beego.AppConfig.String("excel::v_pack_20_03_url") - du = beego.AppConfig.DefaultString("excel::v_pack_20_03_url", du) - localPath := filepath.Join(dir, affectBranch+"_release-package.CSV") - err = downloadPackageFile(localPath, du) - if err != nil { - logs.Error(err) - fr.State = 2 - _ = fr.Update("state") - return - } - pkgList, err := ExtractPackageData(localPath) + pkgList, err := ExtractPackageData(localPath) + if err != nil { + logs.Error(err) + fr.State = 2 + } else { + su := time.Now().Format("2006-01-02") + snPrefix := "openEuler-" + su + snSuffix := int64(1001) + err = GenerateCveExcelByTrigger(affectBranch, fileName, snPrefix, startTime, + snSuffix, true, pkgList, cvrfFileList, componentMap, cvfrFileMap) if err != nil { logs.Error(err) fr.State = 2 - } else { - su := time.Now().Format("2006-01-02") - snPrefix := "openEuler-" + su - snSuffix := int64(1001) - err = GenerateCveExcelByTrigger(affectBranch, fileName, snPrefix, startTime, snSuffix, true, pkgList) - if err != nil { - logs.Error(err) - fr.State = 2 - } - fr.State = 1 } - _ = fr.Update("state") - } else { - logs.Error("affectBranch: ", affectBranch, ", 不能导出sa") + fr.State = 1 } + _ = fr.Update("state") + // zip file zipFileList := []string{fileName, filePath} //zipFileList := []string{"download/cve与安全公告openEuler-20.03-LTS_1607081986.xlsx", "download/updateinfo.xml"} @@ -1923,7 +1914,7 @@ func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { logs.Error("File compression failed: err: ", zipErr) } // send email - sendError := SendEmail(zipFileName) + sendError := SendEmail(zipFileName, 2) if sendError != nil { logs.Error("SendEmail, sendErr: ", sendError) return @@ -1931,7 +1922,7 @@ func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { zipFileList = append(zipFileList, zipFileName) } // Clear file - delFile(zipFileList) + DelFile(zipFileList) logs.Info(affectBranch, ", GenerateExcelTrigger: end") } diff --git a/taskhandler/cvrf.go b/taskhandler/cvrf.go new file mode 100644 index 0000000000000000000000000000000000000000..eb033a0a43c308dc55982288453e6079bf4577ac --- /dev/null +++ b/taskhandler/cvrf.go @@ -0,0 +1,1167 @@ +package taskhandler + +import ( + "bytes" + "compress/zlib" + "cvevulner/common" + "cvevulner/models" + "encoding/xml" + "errors" + "fmt" + "github.com/astaxie/beego/logs" + "io/ioutil" + "os" + "sort" + "strconv" + "strings" +) + +type CvrfSa struct { + XMLName xml.Name `xml:"cvrfdoc,omitempty"` + Xmlns string `xml:"xmlns,attr"` + XmlnsCvrf string `xml:"xmlns:cvrf,attr"` + DocumentTitle *DocumentTitle `xml:"DocumentTitle,omitempty"` + DocumentType string `xml:"DocumentType"` + DocumentPublisher *DocumentPublisher `xml:"DocumentPublisher,omitempty"` + DocumentTracking *DocumentTracking `xml:"DocumentTracking,omitempty"` + DocumentNotes *DocumentNotes `xml:"DocumentNotes,omitempty"` + DocumentReferences *DocumentReferences `xml:"DocumentReferences,omitempty"` + ProductTree *ProductTree `xml:"ProductTree,omitempty"` + Vulnerability []Vulnerability `xml:"Vulnerability,omitempty"` +} + +type DocumentTitle struct { + XMLName xml.Name `xml:"DocumentTitle,omitempty"` + XmlLang string `xml:"xml:lang,attr"` + DocumentTitle string `xml:",innerxml"` +} + +type DocumentPublisher struct { + XMLName xml.Name `xml:"DocumentPublisher,omitempty"` + Type string `xml:"Type,attr"` + ContactDetails string `xml:"ContactDetails"` + IssuingAuthority string `xml:"IssuingAuthority"` +} + +type DocumentTracking struct { + XMLName xml.Name `xml:"DocumentTracking,omitempty"` + Identification *Identification `xml:"Identification,omitempty"` + Status string `xml:"Status"` + Version string `xml:"Version"` + RevisionHistory *RevisionHistory `xml:"RevisionHistory,omitempty"` + InitialReleaseDate string `xml:"InitialReleaseDate"` + CurrentReleaseDate string `xml:"CurrentReleaseDate"` + Generator *Generator `xml:"Generator,omitempty"` +} + +type Identification struct { + XMLName xml.Name `xml:"Identification,omitempty"` + Id string `xml:"ID"` +} + +type RevisionHistory struct { + XMLName xml.Name `xml:"RevisionHistory,omitempty"` + Revision []Revision `xml:"Revision,omitempty"` +} + +type Revision struct { + XMLName xml.Name `xml:"Revision,omitempty"` + Number string `xml:"Number"` + Date string `xml:"Date"` + Description string `xml:"Description"` +} + +type Generator struct { + XMLName xml.Name `xml:"Generator,omitempty"` + Engine string `xml:"Engine"` + Date string `xml:"Date"` +} + +type DocumentNotes struct { + XMLName xml.Name `xml:"DocumentNotes,omitempty"` + Note []Note `xml:"Note,omitempty"` +} + +type Note struct { + XMLName xml.Name `xml:"Note,omitempty"` + Title string `xml:"Title,attr"` + Type string `xml:"Type,attr"` + Ordinal string `xml:"Ordinal,attr"` + XmlLang string `xml:"xml:lang,attr"` + Note string `xml:",innerxml"` +} + +type DocumentReferences struct { + XMLName xml.Name `xml:"DocumentReferences,omitempty"` + CveReference []CveReference `xml:"Reference,omitempty"` +} + +type CveReference struct { + XMLName xml.Name `xml:"Reference,omitempty"` + Type string `xml:"Type,attr"` + CveUrl []CveUrl `xml:"URL,omitempty"` +} + +type CveUrl struct { + XMLName xml.Name `xml:"URL,omitempty"` + Url string `xml:",innerxml"` +} + +type ProductTree struct { + XMLName xml.Name `xml:"ProductTree,omitempty"` + Xmlns string `xml:"xmlns,attr"` + OpenEulerBranch []OpenEulerBranch `xml:"Branch,omitempty"` +} + +type OpenEulerBranch struct { + XMLName xml.Name `xml:"Branch,omitempty"` + Type string `xml:"Type,attr"` + Name string `xml:"Name,attr"` + FullProductName []FullProductName `xml:"FullProductName,omitempty"` +} + +type FullProductName struct { + XMLName xml.Name `xml:"FullProductName,omitempty"` + ProductId string `xml:"ProductID,attr"` + Cpe string `xml:"CPE,attr"` + FullProductName string `xml:",innerxml"` +} + +type Vulnerability struct { + XMLName xml.Name `xml:"Vulnerability,omitempty"` + Ordinal string `xml:"Ordinal,attr"` + Xmlns string `xml:"xmlns,attr"` + CveNotes *CveNotes `xml:"Notes,omitempty"` + ReleaseDate string `xml:"ReleaseDate"` + Cve string `xml:"CVE"` + ProductStatuses *ProductStatuses `xml:"ProductStatuses,omitempty"` + Threats *Threats `xml:"Threats,omitempty"` + CvssScoreSets *CVSSScoreSets `xml:"CVSSScoreSets,omitempty"` + Remediations *Remediations `xml:"Remediations,omitempty"` +} + +type CveNotes struct { + XMLName xml.Name `xml:"Notes,omitempty"` + CveNote *CveNote `xml:"Note,omitempty"` +} + +type CveNote struct { + XMLName xml.Name `xml:"Note,omitempty"` + Title string `xml:"Title,attr"` + Type string `xml:"Type,attr"` + Ordinal string `xml:"Ordinal,attr"` + XmlLnag string `xml:"xml:lang,attr"` + Note string `xml:",innerxml"` +} + +type ProductStatuses struct { + XMLName xml.Name `xml:"ProductStatuses,omitempty"` + Status *Status `xml:"Status,omitempty"` +} + +type Status struct { + XMLName xml.Name `xml:"Status,omitempty"` + Type string `xml:"Type,attr"` + ProductId []ProductId `xml:"ProductID,omitempty"` +} + +type ProductId struct { + XMLName xml.Name `xml:"ProductID,omitempty"` + ProductId string `xml:",innerxml"` +} + +type Threats struct { + XMLName xml.Name `xml:"Threats,omitempty"` + Threat *Threat `xml:"Threat,omitempty"` +} + +type Threat struct { + XMLName xml.Name `xml:"Threat,omitempty"` + Type string `xml:"Type,attr"` + Description string `xml:"Description"` +} + +type CVSSScoreSets struct { + XMLName xml.Name `xml:"CVSSScoreSets,omitempty"` + ScoreSet *ScoreSet `xml:"ScoreSet,omitempty"` +} + +type ScoreSet struct { + XMLName xml.Name `xml:"ScoreSet,omitempty"` + BaseScore string `xml:"BaseScore"` + Vector string `xml:"Vector"` +} + +type Remediations struct { + XMLName xml.Name `xml:"Remediations,omitempty"` + Remediation *Remediation `xml:"Remediation,omitempty"` +} + +type UnRemediations struct { + XMLName xml.Name `xml:"Remediations,omitempty"` + Remediation *UnRemediation `xml:"Remediation,omitempty"` +} + +type Remediation struct { + XMLName xml.Name `xml:"Remediation,omitempty"` + Type string `xml:"Type,attr"` + Description string `xml:"Description"` + Date string `xml:"DATE"` + Url string `xml:"URL"` +} + +type UnRemediation struct { + XMLName xml.Name `xml:"Remediation,omitempty"` + Type string `xml:"Type,attr"` + Description string `xml:"Description"` + Date string `xml:"DATE"` +} + +type CveInfo struct { + CveNum string + Description string + ProductID string + CveLevel string + BaseScore float64 + Vector string + Summary string + OpenEulerSANumUrl string +} + +// Intermediate information +type SecurityNoticeCvrf struct { + CveNum string + Summary string + Theme string + ReferenceLink string + Description string + Introduction string + CveLevel string + BaseScore float64 +} + +type PackRpmCvrf struct { + PackName string +} + +type CveCvrf struct { + Introduction string + OpenEulerSANum string + PublicDate string + CveLevel string + OwnedComponent string + SecurityNotice map[string][]SecurityNoticeCvrf + PackRpmx map[string][]PackRpm + CveInfo CveInfo +} + +type ComponentInfo struct { + OpenEulerSANum string + OwnedComponent string + CveNum []string + CveNumMap map[string][]string + OpenEulerScore []float64 + UpdateFlag int +} + +type UnaffectCvrfSa struct { + XMLName xml.Name `xml:"cvrfdoc,omitempty"` + Xmlns string `xml:"xmlns,attr"` + XmlnsCvrf string `xml:"xmlns:cvrf,attr"` + Vulnerability []UnaffectVulnerability `xml:"Vulnerability,omitempty"` +} + +type UnaffectVulnerability struct { + XMLName xml.Name `xml:"Vulnerability,omitempty"` + Ordinal string `xml:"Ordinal,attr"` + Xmlns string `xml:"xmlns,attr"` + CveNotes *CveNotes `xml:"Notes,omitempty"` + Cve string `xml:"CVE"` + ProductStatuses *ProductStatuses `xml:"ProductStatuses,omitempty"` + CvssScoreSets *CVSSScoreSets `xml:"CVSSScoreSets,omitempty"` + Remediations *UnRemediations `xml:"Remediations,omitempty"` +} + +func UnaffectReadCvrfXml(filePath string, unaffectCvrfSa *UnaffectCvrfSa) error { + if filePath == "" || len(filePath) == 0 { + logs.Error("read err: ", filePath) + return errors.New("file does not exist") + } + fisExist, ferr := PathExists(filePath) + if !fisExist { + logs.Error(ferr) + return ferr + } + fd, err := os.Open(filePath) + if err != nil { + logs.Error("open file err : ", err, ",filePath: ", filePath) + return err + } + defer fd.Close() + fileContent, err := ioutil.ReadAll(fd) + if err != nil { + logs.Error("read file err : ", err, ", filePath: ", filePath) + return err + } + err = xml.Unmarshal(fileContent, unaffectCvrfSa) + if err != nil { + logs.Error("unmarshal err : ", err, ", fileContent: ", fileContent) + return err + } + return nil +} + +func WriteUnaffectCvrfXml(filePath string, unaffectCvrfsa *UnaffectCvrfSa) { + os.Remove(filePath) + xmlOutPut, outPutErr := xml.MarshalIndent(unaffectCvrfsa, "", " ") + if outPutErr == nil { + headerBytes := []byte(xml.Header) + xmlOutPutData := append(headerBytes, xmlOutPut...) + ioutil.WriteFile(filePath, xmlOutPutData, os.ModeAppend) + } else { + logs.Error(outPutErr) + } +} + +func BuildUnaffectCvrfXml(unaffectCvrfsa *UnaffectCvrfSa, v models.ExcelExport, + affectBranch string, componentMap map[string]ComponentInfo) { + v.Description = strings.ReplaceAll(v.Description, "\n\n", " ") + v.Theme = strings.ReplaceAll(v.Theme, "\n\n", " ") + unaffectCvrfsa.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" + unaffectCvrfsa.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" + BuildUnaffectVulnerabilitySet(unaffectCvrfsa, v, affectBranch, componentMap) +} + +func ReadCvrfXml(filePath string, cvrf *CvrfSa) error { + if filePath == "" || len(filePath) == 0 { + logs.Error("read err: ", filePath) + return errors.New("file does not exist") + } + fisExist, ferr := PathExists(filePath) + if !fisExist { + logs.Error(ferr) + return ferr + } + fd, err := os.Open(filePath) + if err != nil { + logs.Error("open file err : ", err, ",filePath: ", filePath) + return err + } + defer fd.Close() + fileContent, err := ioutil.ReadAll(fd) + if err != nil { + logs.Error("read file err : ", err, ", filePath: ", filePath) + return err + } + err = xml.Unmarshal(fileContent, cvrf) + if err != nil { + logs.Error("unmarshal err : ", err, ", fileContent: ", fileContent) + return err + } + return nil +} + +func WriteCvrfXml(filePath string, cvrfsa *CvrfSa) { + os.Remove(filePath) + xmlOutPut, outPutErr := xml.MarshalIndent(cvrfsa, "", " ") + if outPutErr == nil { + headerBytes := []byte(xml.Header) + xmlOutPutData := append(headerBytes, xmlOutPut...) + ioutil.WriteFile(filePath, xmlOutPutData, os.ModeAppend) + } else { + logs.Error(outPutErr) + } +} + +func BuildUnaffectVulnerabilitySet(unaffectCvrfsa *UnaffectCvrfSa, v models.ExcelExport, + affectBranch string, componentMap map[string]ComponentInfo) { + vulnerability := make([]UnaffectVulnerability, 0) + if unaffectCvrfsa.Vulnerability != nil && len(unaffectCvrfsa.Vulnerability) > 0 { + for _, vuln := range unaffectCvrfsa.Vulnerability { + vulnerability = append(vulnerability, vuln) + } + } + vulnerabilityx := BuildUnaffVulnerabilitySlice(vulnerability, v, affectBranch, componentMap) + unaffectCvrfsa.Vulnerability = vulnerabilityx +} + +func BuildUnaffVulnerabilitySlice(vulnerability []UnaffectVulnerability, v models.ExcelExport, + affectBranch string, componentMap map[string]ComponentInfo) []UnaffectVulnerability { + affectBranchListx := strings.Split(affectBranch, "-") + cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", + affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + if vulnerability != nil && len(vulnerability) > 0 { + cveExist := false + for _, vl := range vulnerability { + if vl.Cve == v.CveNum && vl.Remediations.Remediation.Description == v.InfluenceComponent { + cpeExist := false + for _, pid := range vl.ProductStatuses.Status.ProductId { + if pid.ProductId == cpe { + cpeExist = true + break + } + } + if !cpeExist { + var productId ProductId + productId.ProductId = cpe + vl.ProductStatuses.Status.ProductId = append(vl.ProductStatuses.Status.ProductId, productId) + } + cveExist = true + break + } + } + if !cveExist { + vlLenth := len(vulnerability) + 1 + vulnerabilitySlice := BuildUnaffVulnerability(vlLenth, v, componentMap, cpe) + if len(vulnerabilitySlice) > 0 { + vulnerability = append(vulnerability, vulnerabilitySlice...) + } + } + } else { + vlLenth := 1 + vulnerabilitySlice := BuildUnaffVulnerability(vlLenth, v, componentMap, cpe) + vulnerability = append(vulnerability, vulnerabilitySlice...) + } + return vulnerability +} + +func BuildUnaffVulnerability(vlLenth int, v models.ExcelExport, + componentMap map[string]ComponentInfo, cpe string) []UnaffectVulnerability { + vulnerabilitySlice := make([]UnaffectVulnerability, 0) + var vulnerability UnaffectVulnerability + vulnerability.Xmlns = "http://www.icasi.org/CVRF/schema/vuln/1.1" + vulnerability.Ordinal = strconv.Itoa(vlLenth) + var cveNotes CveNotes + var cveNote CveNote + cveNote.Ordinal = strconv.Itoa(vlLenth) + cveNote.Type = "General" + cveNote.Title = "Vulnerability Description" + cveNote.Note = v.CveBrief + cveNote.XmlLnag = "en" + cveNotes.CveNote = &cveNote + vulnerability.CveNotes = &cveNotes + vulnerability.Cve = v.CveNum + var productStatuses ProductStatuses + var status Status + status.Type = "Unaffected" + var productId ProductId + productId.ProductId = cpe + productIdSlice := make([]ProductId, 0) + productIdSlice = append(productIdSlice, productId) + status.ProductId = productIdSlice + productStatuses.Status = &status + vulnerability.ProductStatuses = &productStatuses + var cVSSScoreSets CVSSScoreSets + var scoreSet ScoreSet + scoreSet.BaseScore = fmt.Sprintf("%.1f", v.OpenEulerScore) + scoreSet.Vector = v.OvectorVule + cVSSScoreSets.ScoreSet = &scoreSet + vulnerability.CvssScoreSets = &cVSSScoreSets + var remediations UnRemediations + var remediation UnRemediation + remediation.Type = "Unaffected" + remediation.Description = v.InfluenceComponent + remediation.Date = common.GetCurDate() + remediations.Remediation = &remediation + vulnerability.Remediations = &remediations + vulnerabilitySlice = append(vulnerabilitySlice, vulnerability) + return vulnerabilitySlice +} + +func BranchExist(affectBranch string, cvrfFileList map[string][]string) []string { + brancsList, keyOk := cvrfFileList[BRANCHSKEY] + if !keyOk { + localBran := make([]string, 0) + localBran = append(localBran, affectBranch) + cvrfFileList[BRANCHSKEY] = localBran + } else { + brlFlag := false + for _, brl := range brancsList { + if brl == affectBranch { + brlFlag = true + break + } + } + if !brlFlag { + brancsList = append(brancsList, affectBranch) + cvrfFileList[BRANCHSKEY] = brancsList + } + } + brancsListx, _ := cvrfFileList[BRANCHSKEY] + if len(brancsListx) > 1 { + sort.Strings(brancsListx) + } + return brancsListx +} + +func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, + introduction, topic, affectBranch string, + componentMap map[string]ComponentInfo, + branchList []string) { + documentNotes := cvrfsa.DocumentNotes + if documentNotes == nil || cvrfsa.DocumentNotes.Note == nil || + len(cvrfsa.DocumentNotes.Note) == 0 { + var documentNotesx DocumentNotes + note := make([]Note, 0) + var noteSynopsis Note + noteSynopsis.Title = "Synopsis" + noteSynopsis.Type = "General" + noteSynopsis.Ordinal = "1" + noteSynopsis.XmlLang = "en" + noteSynopsis.Note = v.Summary + note = append(note, noteSynopsis) + var noteSummary Note + noteSummary.Title = "Summary" + noteSummary.Type = "General" + noteSummary.Ordinal = "2" + noteSummary.XmlLang = "en" + noteSummary.Note = introduction + note = append(note, noteSummary) + var noteDescription Note + noteDescription.Title = "Description" + noteDescription.Type = "General" + noteDescription.Ordinal = "3" + noteDescription.XmlLang = "en" + noteDescription.Note = v.Description + note = append(note, noteDescription) + var noteTopic Note + noteTopic.Title = "Topic" + noteTopic.Type = "General" + noteTopic.Ordinal = "4" + noteTopic.XmlLang = "en" + noteTopic.Note = topic + note = append(note, noteTopic) + var noteSeverity Note + noteSeverity.Title = "Severity" + noteSeverity.Type = "General" + noteSeverity.Ordinal = "5" + noteSeverity.XmlLang = "en" + cveLevel := openEulerScoreProc(v.OpenEulerScore) + noteSeverity.Note = cveLevel + note = append(note, noteSeverity) + var noteComponent Note + noteComponent.Title = "Affected Component" + noteComponent.Type = "General" + noteComponent.Ordinal = "6" + noteComponent.XmlLang = "en" + noteComponent.Note = v.InfluenceComponent + note = append(note, noteComponent) + documentNotesx.Note = note + cvrfsa.DocumentNotes = &documentNotesx + } else { + branchCount := len(branchList) + note := cvrfsa.DocumentNotes.Note + ownedComponent := "" + if len(v.InfluenceComponent) > 1 { + ownedComponent = v.InfluenceComponent + } else { + ownedComponent = v.OwnedComponent + } + componentInfo := componentMap[ownedComponent] + notex := make([]Note, 0) + for _, te := range note { + if te.Title == "Description" { + dSplit := strings.Split(v.Description, "Security Fix(es):") + if len(dSplit) > 1 { + if !strings.Contains(te.Note, dSplit[0]) { + te.Note = dSplit[0] + te.Note + } + if !strings.Contains(te.Note, dSplit[1]) { + te.Note += dSplit[1] + } + } + te.Note = te.Note + } + if te.Title == "Topic" { + vcn := strings.Join(componentInfo.CveNum, ";\n") + theme, err := models.GetCanExportTheme(vcn, v.InfluenceComponent, affectBranch) + if err == nil && len(theme) > 1 { + if branchCount <= 1 { + te.Note = theme + } else if branchCount == 2 { + te.Note = strings.ReplaceAll(theme, affectBranch, strings.Join(branchList, " and ")) + } else { + reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] + te.Note = strings.ReplaceAll(theme, affectBranch, reBanch) + } + } + } + if te.Title == "Summary" { + if branchCount <= 1 { + te.Note = v.Introduction + } else if branchCount == 2 { + te.Note = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) + } else { + reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] + te.Note = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) + } + } + if te.Title == "Severity" { + openEulerScoreSlice := componentInfo.OpenEulerScore + if len(openEulerScoreSlice) > 1 { + sort.Float64s(openEulerScoreSlice) + } + if len(openEulerScoreSlice) > 0 { + cveLevel := openEulerScoreProc(openEulerScoreSlice[len(openEulerScoreSlice)-1]) + te.Note = cveLevel + } + } + notex = append(notex, te) + } + cvrfsa.DocumentNotes.Note = notex + } +} + +func BuildDocumentTitle(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, + cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, + curDate string, branchList []string) { + var documt DocumentTitle + introduction := "" + topic := "" + branchCount := len(branchList) + if branchCount <= 1 { + introduction = v.Introduction + topic = v.Theme + } else if branchCount == 2 { + introduction = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) + topic = strings.ReplaceAll(v.Theme, affectBranch, strings.Join(branchList, " and ")) + } else { + reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] + introduction = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) + topic = strings.ReplaceAll(v.Theme, affectBranch, reBanch) + } + if len(introduction) > 1 { + documt.DocumentTitle = introduction[:len(introduction)-1] + } else { + documt.DocumentTitle = introduction + } + documt.XmlLang = "en" + cvrfsa.DocumentTitle = &documt + cvrfsa.DocumentType = "Security Advisory" + var documentPublisher DocumentPublisher + documentPublisher.Type = "Vendor" + documentPublisher.ContactDetails = "openeuler-security@openeuler.org" + documentPublisher.IssuingAuthority = "openEuler security committee" + cvrfsa.DocumentPublisher = &documentPublisher + var documentTracking DocumentTracking + var identification Identification + ownedComponent := "" + if len(v.InfluenceComponent) > 1 { + ownedComponent = v.InfluenceComponent + } else { + ownedComponent = v.OwnedComponent + } + componentInfo := componentMap[ownedComponent] + identification.Id = componentInfo.OpenEulerSANum + documentTracking.Identification = &identification + documentTracking.Status = "Final" + documentTracking.Version = "1.0" + var revisionHistory RevisionHistory + revision := make([]Revision, 0) + if cvrfsa.DocumentTracking != nil && + cvrfsa.DocumentTracking.RevisionHistory != nil && + len(cvrfsa.DocumentTracking.RevisionHistory.Revision) > 0 && + componentInfo.UpdateFlag == 1 { + RevisionSlice := cvrfsa.DocumentTracking.RevisionHistory.Revision + for _, rev := range RevisionSlice { + revision = append(revision, rev) + } + versionValue := float64(len(RevisionSlice))/float64(10) + float64(1) + var revisionx Revision + revisionx.Number = fmt.Sprintf("%.1f", versionValue) + revisionx.Date = v.PublicDate + revisionx.Description = "Update" + revision = append(revision, revisionx) + documentTracking.Version = revisionx.Number + } else { + var revisionx Revision + revisionx.Number = "1.0" + revisionx.Date = v.PublicDate + revisionx.Description = "Initial" + revision = append(revision, revisionx) + documentTracking.InitialReleaseDate = v.PublicDate + } + revisionHistory.Revision = revision + documentTracking.RevisionHistory = &revisionHistory + documentTracking.CurrentReleaseDate = v.PublicDate + var generator Generator + generator.Date = curDate + generator.Engine = "openEuler SA Tool V1.0" + documentTracking.Generator = &generator + cvrfsa.DocumentTracking = &documentTracking + BuilddocumentNotes(cvrfsa, v, introduction, topic, + affectBranch, componentMap, branchList) +} + +func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, componentMap map[string]ComponentInfo) { + componentInfo, comOk := componentMap[v.InfluenceComponent] + if comOk { + if cvrfsa.DocumentReferences != nil && len(cvrfsa.DocumentReferences.CveReference) > 0 { + cveReference := cvrfsa.DocumentReferences.CveReference + cveReferencex := make([]CveReference, 0) + for _, cveRef := range cveReference { + if cveRef.Type == "openEuler CVE" { + isExist := false + for _, cUrl := range cveRef.CveUrl { + if strings.Contains(cUrl.Url, v.CveNum) { + isExist = true + } + } + if !isExist { + var cveUrl1 CveUrl + cveUrl1.Url = "https://openeuler.org/en/security/cve/detail.html?id=" + v.CveNum + cveRef.CveUrl = append(cveRef.CveUrl, cveUrl1) + } + } + if cveRef.Type == "Other" { + isExist := false + for _, cUrl := range cveRef.CveUrl { + if strings.Contains(cUrl.Url, v.CveNum) { + isExist = true + } + } + if !isExist { + var cveUrl2 CveUrl + cveUrl2.Url = "https://nvd.nist.gov/vuln/detail/" + v.CveNum + cveRef.CveUrl = append(cveRef.CveUrl, cveUrl2) + } + } + cveReferencex = append(cveReferencex, cveRef) + } + cvrfsa.DocumentReferences.CveReference = cveReferencex + } else { + var documentReferences DocumentReferences + cveReferenceSlice := make([]CveReference, 0) + var cveReference0 CveReference + cveUrlSlice0 := make([]CveUrl, 0) + var cveUrl0 CveUrl + cveUrl0.Url = "https://openeuler.org/en/security/safety-bulletin/detail.html?id=" + componentInfo.OpenEulerSANum + cveUrlSlice0 = append(cveUrlSlice0, cveUrl0) + cveReference0.Type = "Self" + cveReference0.CveUrl = cveUrlSlice0 + cveReferenceSlice = append(cveReferenceSlice, cveReference0) + var cveReference1 CveReference + cveUrlSlice1 := make([]CveUrl, 0) + for _, cveNum := range componentInfo.CveNum { + var cveUrl1 CveUrl + cveUrl1.Url = "https://openeuler.org/en/security/cve/detail.html?id=" + cveNum + cveUrlSlice1 = append(cveUrlSlice1, cveUrl1) + } + cveReference1.Type = "openEuler CVE" + cveReference1.CveUrl = cveUrlSlice1 + cveReferenceSlice = append(cveReferenceSlice, cveReference1) + var cveReference2 CveReference + cveUrlSlice2 := make([]CveUrl, 0) + for _, cveNum := range componentInfo.CveNum { + var cveUrl2 CveUrl + cveUrl2.Url = "https://nvd.nist.gov/vuln/detail/" + cveNum + cveUrlSlice2 = append(cveUrlSlice2, cveUrl2) + } + cveReference2.Type = "Other" + cveReference2.CveUrl = cveUrlSlice2 + cveReferenceSlice = append(cveReferenceSlice, cveReference2) + documentReferences.CveReference = cveReferenceSlice + cvrfsa.DocumentReferences = &documentReferences + } + } +} + +func BuildProductTree(cvrfsa *CvrfSa, v models.ExcelExport, pkg []models.Package, affectBranch string) { + if cvrfsa.ProductTree != nil && len(cvrfsa.ProductTree.OpenEulerBranch) > 0 { + affectBranchListx := strings.Split(affectBranch, "-") + cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", + affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + openEulerBranchx := make([]OpenEulerBranch, 0) + for _, opBranch := range cvrfsa.ProductTree.OpenEulerBranch { + if opBranch.Type == "Product Name" && strings.Contains(affectBranch, opBranch.Name) { + fpFlag := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == affectBranch { + fpFlag = true + break + } + } + if !fpFlag { + var fullProductName0 FullProductName + fullProductName0.FullProductName = affectBranch + fullProductName0.ProductId = affectBranch + fullProductName0.Cpe = cpe + opBranch.FullProductName = append(opBranch.FullProductName, fullProductName0) + } + } + if opBranch.Type == "Package Arch" { + fullProductNameSliceaarch64 := make([]FullProductName, 0) + fullProductNameSlicenoarch := make([]FullProductName, 0) + fullProductNameSlicesrc := make([]FullProductName, 0) + fullProductNameSlicex86_64 := make([]FullProductName, 0) + if len(pkg) > 0 { + for _, pk := range pkg { + if opBranch.Name == "aarch64" { + isExist := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == pk.PackName { + isExist = true + break + } + } + if !isExist && strings.Contains(pk.PackName, ".aarch64.") { + var fullProductNameaarch64 FullProductName + fullProductNameaarch64.FullProductName = pk.PackName + fullProductNameaarch64.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNameaarch64.ProductId = pk.PackName[:index] + } + fullProductNameSliceaarch64 = append(fullProductNameSliceaarch64, fullProductNameaarch64) + } + } else if opBranch.Name == "noarch" { + isExist := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == pk.PackName { + isExist = true + break + } + } + if !isExist && strings.Contains(pk.PackName, ".noarch.") { + var fullProductNamenoarch FullProductName + fullProductNamenoarch.FullProductName = pk.PackName + fullProductNamenoarch.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamenoarch.ProductId = pk.PackName[:index] + } + fullProductNameSlicenoarch = append(fullProductNameSlicenoarch, fullProductNamenoarch) + } + } else if opBranch.Name == "x86_64" { + isExist := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == pk.PackName { + isExist = true + break + } + } + if !isExist && strings.Contains(pk.PackName, ".x86_64.") { + var fullProductNamex86_64 FullProductName + fullProductNamex86_64.FullProductName = pk.PackName + fullProductNamex86_64.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamex86_64.ProductId = pk.PackName[:index] + } + fullProductNameSlicex86_64 = append(fullProductNameSlicex86_64, fullProductNamex86_64) + } + } else { + isExist := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == pk.PackName { + isExist = true + break + } + } + if !isExist && strings.Contains(pk.PackName, ".src.") { + var fullProductNamesrc FullProductName + fullProductNamesrc.FullProductName = pk.PackName + fullProductNamesrc.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamesrc.ProductId = pk.PackName[:index] + } + fullProductNameSlicesrc = append(fullProductNameSlicesrc, fullProductNamesrc) + } + } + } + } + if opBranch.Name == "aarch64" { + opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSliceaarch64...) + } else if opBranch.Name == "noarch" { + opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSlicenoarch...) + } else if opBranch.Name == "x86_64" { + opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSlicex86_64...) + } else { + opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSlicesrc...) + } + } + openEulerBranchx = append(openEulerBranchx, opBranch) + } + cvrfsa.ProductTree.OpenEulerBranch = openEulerBranchx + } else { + var productTree ProductTree + productTree.Xmlns = "http://www.icasi.org/CVRF/schema/prod/1.1" + openEulerBranchSlice := make([]OpenEulerBranch, 0) + var openEulerBranch0 OpenEulerBranch + fullProductNameSlice0 := make([]FullProductName, 0) + var fullProductName0 FullProductName + fullProductName0.FullProductName = affectBranch + fullProductName0.ProductId = affectBranch + affectBranchListx := strings.Split(affectBranch, "-") + cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", + affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + fullProductName0.Cpe = cpe + fullProductNameSlice0 = append(fullProductNameSlice0, fullProductName0) + openEulerBranch0.FullProductName = fullProductNameSlice0 + openEulerBranch0.Type = "Product Name" + openEulerBranch0.Name = affectBranchListx[0] + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranch0) + fullProductNameSliceaarch64 := make([]FullProductName, 0) + fullProductNameSlicenoarch := make([]FullProductName, 0) + fullProductNameSlicesrc := make([]FullProductName, 0) + fullProductNameSlicex86_64 := make([]FullProductName, 0) + if len(pkg) > 0 { + for _, pk := range pkg { + if strings.Contains(pk.PackName, ".aarch64.") { + var fullProductNameaarch64 FullProductName + fullProductNameaarch64.FullProductName = pk.PackName + fullProductNameaarch64.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNameaarch64.ProductId = pk.PackName[:index] + } + fullProductNameSliceaarch64 = append(fullProductNameSliceaarch64, fullProductNameaarch64) + } else if strings.Contains(pk.PackName, ".src.") { + var fullProductNamesrc FullProductName + fullProductNamesrc.FullProductName = pk.PackName + fullProductNamesrc.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamesrc.ProductId = pk.PackName[:index] + } + fullProductNameSlicesrc = append(fullProductNameSlicesrc, fullProductNamesrc) + } else if strings.Contains(pk.PackName, ".x86_64.") { + var fullProductNamex86_64 FullProductName + fullProductNamex86_64.FullProductName = pk.PackName + fullProductNamex86_64.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamex86_64.ProductId = pk.PackName[:index] + } + fullProductNameSlicex86_64 = append(fullProductNameSlicex86_64, fullProductNamex86_64) + } else { + var fullProductNamenoarch FullProductName + fullProductNamenoarch.FullProductName = pk.PackName + fullProductNamenoarch.Cpe = cpe + index := strings.LastIndex(pk.PackName, ".oe1.") + if index > 0 { + fullProductNamenoarch.ProductId = pk.PackName[:index] + } + fullProductNameSlicenoarch = append(fullProductNameSlicenoarch, fullProductNamenoarch) + } + } + } + if len(fullProductNameSliceaarch64) > 0 { + var openEulerBranchaarch64 OpenEulerBranch + openEulerBranchaarch64.Type = "Package Arch" + openEulerBranchaarch64.Name = "aarch64" + openEulerBranchaarch64.FullProductName = fullProductNameSliceaarch64 + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchaarch64) + } + if len(fullProductNameSlicenoarch) > 0 { + var openEulerBranchnoarch OpenEulerBranch + openEulerBranchnoarch.Type = "Package Arch" + openEulerBranchnoarch.Name = "noarch" + openEulerBranchnoarch.FullProductName = fullProductNameSlicenoarch + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchnoarch) + } + if len(fullProductNameSlicesrc) > 0 { + var openEulerBranchsrc OpenEulerBranch + openEulerBranchsrc.Type = "Package Arch" + openEulerBranchsrc.Name = "src" + openEulerBranchsrc.FullProductName = fullProductNameSlicesrc + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchsrc) + } + if len(fullProductNameSlicex86_64) > 0 { + var openEulerBranchx86_64 OpenEulerBranch + openEulerBranchx86_64.Type = "Package Arch" + openEulerBranchx86_64.Name = "x86_64" + openEulerBranchx86_64.FullProductName = fullProductNameSlicex86_64 + openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchx86_64) + } + productTree.OpenEulerBranch = openEulerBranchSlice + cvrfsa.ProductTree = &productTree + } +} + +func BuildVulnerability(vlLenth int, v models.ExcelExport, + componentMap map[string]ComponentInfo, cpe string) []Vulnerability { + vulnerabilitySlice := make([]Vulnerability, 0) + var vulnerability Vulnerability + vulnerability.Xmlns = "http://www.icasi.org/CVRF/schema/vuln/1.1" + vulnerability.Ordinal = strconv.Itoa(vlLenth) + var cveNotes CveNotes + var cveNote CveNote + cveNote.Ordinal = strconv.Itoa(vlLenth) + cveNote.Type = "General" + cveNote.Title = "Vulnerability Description" + cveNote.Note = v.CveBrief + cveNote.XmlLnag = "en" + cveNotes.CveNote = &cveNote + vulnerability.CveNotes = &cveNotes + vulnerability.ReleaseDate = v.PublicDate + vulnerability.Cve = v.CveNum + var productStatuses ProductStatuses + var status Status + status.Type = "Fixed" + var productId ProductId + productId.ProductId = cpe + productIdSlice := make([]ProductId, 0) + productIdSlice = append(productIdSlice, productId) + status.ProductId = productIdSlice + productStatuses.Status = &status + vulnerability.ProductStatuses = &productStatuses + var threats Threats + var threat Threat + threat.Type = "Impact" + threat.Description = openEulerScoreProc(v.OpenEulerScore) + threats.Threat = &threat + vulnerability.Threats = &threats + var cVSSScoreSets CVSSScoreSets + var scoreSet ScoreSet + scoreSet.BaseScore = fmt.Sprintf("%.1f", v.OpenEulerScore) + scoreSet.Vector = v.OvectorVule + cVSSScoreSets.ScoreSet = &scoreSet + vulnerability.CvssScoreSets = &cVSSScoreSets + var remediations Remediations + var remediation Remediation + remediation.Type = "Vendor Fix" + remediation.Description = v.Summary + remediation.Date = v.PublicDate + ownedComponent := "" + if len(v.InfluenceComponent) > 1 { + ownedComponent = v.InfluenceComponent + } else { + ownedComponent = v.OwnedComponent + } + componentInfo := componentMap[ownedComponent] + remediation.Url = "https://openeuler.org/en/security/safety-bulletin/detail.html?id=" + componentInfo.OpenEulerSANum + remediations.Remediation = &remediation + vulnerability.Remediations = &remediations + vulnerabilitySlice = append(vulnerabilitySlice, vulnerability) + return vulnerabilitySlice +} + +func BuildVulnerabilitySlice(vulnerability []Vulnerability, v models.ExcelExport, + affectBranch string, componentMap map[string]ComponentInfo) []Vulnerability { + affectBranchListx := strings.Split(affectBranch, "-") + cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", + affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + if vulnerability != nil && len(vulnerability) > 0 { + cveExist := false + for _, vl := range vulnerability { + if vl.Cve == v.CveNum && strings.Contains(vl.Remediations.Remediation.Description, v.InfluenceComponent) { + cpeExist := false + for _, pid := range vl.ProductStatuses.Status.ProductId { + if pid.ProductId == cpe { + cpeExist = true + break + } + } + if !cpeExist { + var productId ProductId + productId.ProductId = cpe + vl.ProductStatuses.Status.ProductId = append(vl.ProductStatuses.Status.ProductId, productId) + } + cveExist = true + break + } + } + if !cveExist { + vlLenth := len(vulnerability) + 1 + vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe) + if len(vulnerabilitySlice) > 0 { + vulnerability = append(vulnerability, vulnerabilitySlice...) + } + } + } else { + vlLenth := 1 + vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe) + vulnerability = append(vulnerability, vulnerabilitySlice...) + } + return vulnerability +} + +func BuildVulnerabilitySet(cvrfsa *CvrfSa, v models.ExcelExport, + affectBranch string, componentMap map[string]ComponentInfo) { + vulnerability := make([]Vulnerability, 0) + if cvrfsa.Vulnerability != nil && len(cvrfsa.Vulnerability) > 0 { + for _, vuln := range cvrfsa.Vulnerability { + vulnerability = append(vulnerability, vuln) + } + } + vulnerabilityx := BuildVulnerabilitySlice(vulnerability, v, + affectBranch, componentMap) + cvrfsa.Vulnerability = vulnerabilityx +} + +func BuildCvrfXml(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, + cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, + pkg []models.Package) { + curDate := common.GetCurDate() + v.Description = strings.ReplaceAll(v.Description, "\n\n", " ") + v.Theme = strings.ReplaceAll(v.Theme, "\n\n", " ") + branchList := BranchExist(affectBranch, cvrfFileList) + cvrfsa.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" + cvrfsa.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" + BuildDocumentTitle(cvrfsa, v, affectBranch, cvrfFileList, componentMap, curDate, branchList) + BuildDocumentRef(cvrfsa, v, componentMap) + BuildProductTree(cvrfsa, v, pkg, affectBranch) + BuildVulnerabilitySet(cvrfsa, v, affectBranch, componentMap) +} + +func RecordCrvfInfo(fileName, filex string, fixFlag int8) error { + fileBytes, err := ioutil.ReadFile(fileName) + if err != nil { + fmt.Println("ioutil.ReadFile, error : %s", err, fileName) + return err + } + fileStr := string(fileBytes) + fileMd5 := common.EncryptMd5(fileStr) + openEulerSANum := "" + if fixFlag == UNAFFECTFLAG { + openEulerSANum = fileMd5 + } else { + if len(filex) > 5 { + openEulerSANum = filex[5 : len(filex)-4] + } + } + fileContent := fileBytes + var buf bytes.Buffer + compressor, err := zlib.NewWriterLevelDict(&buf, zlib.BestCompression, fileBytes) + if err != nil { + fmt.Println("Compression failed") + } else { + compressor.Write(fileBytes) + compressor.Close() + fileContent = buf.Bytes() + } + fileConStr := baseStdEncode(fileContent) + var cfr models.CvrfSaRecord + cfr.OpenEulerSANum = openEulerSANum + cfr.Md5 = fileMd5 + tbErr := models.GetCvrfRecord(&cfr, "openeuler_sa_num") + if tbErr != nil || cfr.Id == 0 { + cfr.CreateTime = common.GetCurTime() + cfr.Md5 = fileMd5 + cfr.OpenEulerSANum = openEulerSANum + cfr.XmlContent = fileConStr + cfr.Status = 1 + cfr.IsExport = 1 + cfr.CveNum = " " + cfr.PackName = " " + cfr.AffectFlag = fixFlag + num, iErr := models.InsertCvrfRecord(&cfr) + if iErr != nil || num == 0 { + logs.Error("InsertCvrfRecord, ", iErr) + } + } else { + cfr.UpdateTime = common.GetCurTime() + cfr.UpdateMd5 = fileMd5 + cfr.XmlContent = fileConStr + cfr.Status = 1 + cfr.IsExport = 1 + cfr.AffectFlag = fixFlag + uErr := models.UpdateCvrfRecord(&cfr, "UpdateTime", "UpdateMd5", "XmlContent", "Status", "IsExport", "AffectFlag") + if uErr != nil { + logs.Error("UpdateCvrfRecord, ", uErr) + } + } + return nil +} diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 6925537ca8cabb157ffcccfa60bf49a5972f56ec..a04d6441985453a91a392e37744174f8c4ccbbad 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -1,6 +1,7 @@ package taskhandler import ( + "cvevulner/common" "cvevulner/models" "cvevulner/util" "encoding/csv" @@ -15,6 +16,7 @@ import ( "net/http" "os" "path" + "path/filepath" "regexp" "strconv" "strings" @@ -22,6 +24,12 @@ import ( "time" ) +const CVRFFKEY = "cvrfFileKey" +const BRANCHSKEY = "opBranchsKey" +const UNAFFECTCVRFKEY = "unaffectcvrfkey" +const FIXEDFLAGE = 1 +const UNAFFECTFLAG = 2 + //CveExcel Excel export client type CveExcel struct { ExcelName string //excel name @@ -42,11 +50,14 @@ type IssueAndPkg struct { IssueMap map[int64]models.PullRequestIssue IssuePkg string Repo string + IssueId int64 } var fillLock sync.Mutex var wgTrigger sync.WaitGroup +//var cvrfLock sync.Mutex + //GenerateCveExcel Generate Excel documents based on data. //param snPrefix means security notice prefix. //param snSuffix means security notice suffix append start value. @@ -78,7 +89,8 @@ func GenerateCveExcel(excelName, snPrefix string, snSuffix int64, forceRewrite b //GenerateCveExcelByTrigger Generate cve&security notice excel file by trigger func GenerateCveExcelByTrigger(affectBranch, excelName, snPrefix, startTime string, snSuffix int64, - forceRewrite bool, pkgList []models.ExcelPackage) (err error) { + forceRewrite bool, pkgList []models.ExcelPackage, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) (err error) { if len(pkgList) == 0 { return errors.New("No data to export! ") } @@ -86,6 +98,7 @@ func GenerateCveExcelByTrigger(affectBranch, excelName, snPrefix, startTime stri ec := CveExcel{} err = ec.Init(excelName, snPrefix, snSuffix) if err != nil { + logs.Error("excelName: ", excelName, err) return err } mode := ec.InitFileHandle(forceRewrite) @@ -96,7 +109,7 @@ func GenerateCveExcelByTrigger(affectBranch, excelName, snPrefix, startTime stri logs.Error(err) } } - ec.FillContentTrigger(pkgList, startTime, affectBranch) + ec.FillContentTrigger(pkgList, startTime, affectBranch, cvrfFileList, componentMap, cvfrFileMap) return ec.Save(mode) } @@ -379,7 +392,9 @@ func (ec *CveExcel) FillContent(count int64) { } } -func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, affectBranch string) { +func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, + affectBranch string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { pl := len(pkgList) pageSize := 10 pc := pl / 10 @@ -406,7 +421,8 @@ func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, } for i := 0; i < pc; i++ { wgTrigger.Add(1) - go ec.handleGiteData(cd, affectBranch, &cvexml, &dpdates, securityNotice, packRpmx) + go ec.handleGiteData(cd, affectBranch, &cvexml, &dpdates, + securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap) } wgTrigger.Wait() // write xml @@ -426,7 +442,7 @@ func fileExist(fileList []string) bool { return true } -func delFile(fileList []string) { +func DelFile(fileList []string) { if len(fileList) > 0 { for _, filex := range fileList { err := os.Remove(filex) @@ -442,11 +458,14 @@ func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { if err != nil { return err } + var cvrfFileList map[string][]string + var componentMap map[string]ComponentInfo + var cvfrFileMap map[string]CvrfSa lz := len(list) if lz > 0 { for _, v := range list { if v.Num == 1 { - ec.setContentRow(v, "") + ec.setContentRow(v, "", cvrfFileList, componentMap, cvfrFileMap) } else if v.Num > 1 { //1.Obtain issue_tpl according to cve_num, if all issue_status == 2 0r issue_status == 6, then data can be exported list, err := models.GetIssueTplByCveNum(v.CveNum) @@ -485,7 +504,7 @@ func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { ep.AffectProduct = ep.AffectProduct + "\n" + ex.AffectProduct } } - ec.setContentRow(ep, "") + ec.setContentRow(ep, "", cvrfFileList, componentMap, cvfrFileMap) } } } @@ -493,20 +512,50 @@ func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { return nil } +func procUnaffectCvrfData(v models.ExcelExport, + affectBranch string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo) { + fileDir := beego.AppConfig.String("fileDir") + cvrffileName := filepath.Join(fileDir, "cvrf-unaffected-cve-"+common.GetCurDate()+".xml") + cvrfFileSlice, cvrfOk := cvrfFileList[UNAFFECTCVRFKEY] + if cvrfOk && len(cvrfFileSlice) > 0 { + cvrffileName = cvrfFileSlice[0] + } else { + cvrfNameSlice := make([]string, 0) + cvrfNameSlice = append(cvrfNameSlice, cvrffileName) + cvrfFileList[UNAFFECTCVRFKEY] = cvrfNameSlice + } + var unaffectcvrf UnaffectCvrfSa + // Read file content + readErr := UnaffectReadCvrfXml(cvrffileName, &unaffectcvrf) + if readErr != nil { + unaffectcvrf = UnaffectCvrfSa{} + } + BuildUnaffectCvrfXml(&unaffectcvrf, v, affectBranch, componentMap) + WriteUnaffectCvrfXml(cvrffileName, &unaffectcvrf) + //saveCvrfName(cvrfFileList, cvrffileName, UNAFFECTCVRFKEY) +} + func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBranch string, cvexml *[]CveXml, dpdates *Updates, - securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) (err error) { + securityNotice map[string][]SecurityNoticeXml, + packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) (err error) { lz := len(list) if lz > 0 { for _, v := range list { affectBool := affectBrachRep(&v, affectBranch) if !affectBool { logs.Error("Unaffected version, data: ", v) + // Generate unaffected cvrf files + fillLock.Lock() + procUnaffectCvrfData(v, affectBranch, cvrfFileList, componentMap) + fillLock.Unlock() continue } if v.Num == 1 { fillLock.Lock() - ec.setContentRow(v, affectBranch) + ec.setContentRow(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap) fillLock.Unlock() fillLock.Lock() BuildXml(cvexml, &v, securityNotice, packRpmx) @@ -574,9 +623,10 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, } } } - fillLock.Lock() - ec.setContentRow(ep, affectBranch) - fillLock.Unlock() + // Generate cvrf format without processing + //fillLock.Lock() + //ec.setContentRow(ep, affectBranch, cvrfFileList, componentMap, cvfrFileMap) + //fillLock.Unlock() } } } @@ -619,7 +669,143 @@ func addXmlData(canExport []models.ExcelExport, cvexml *[]CveXml, affectBranch s } } -func (ec *CveExcel) setContentRow(v models.ExcelExport, affectBranch string) { +func StoreComponentInfo(componentMap map[string]ComponentInfo, + v models.ExcelExport, affectBranch string) bool { + repFlag := false + influenceComponent := "" + if len(v.InfluenceComponent) > 1 { + influenceComponent = v.InfluenceComponent + } else { + influenceComponent = v.OwnedComponent + } + packNameStruct, pOk := componentMap[influenceComponent] + if !pOk && len(packNameStruct.OpenEulerSANum) < 1{ + var coponentInfo ComponentInfo + coponentInfo.OpenEulerSANum = v.OpenEulerSANum + coponentInfo.OwnedComponent = influenceComponent + coponentInfo.UpdateFlag = 2 + openEulerScoreSlice := make([]float64, 0) + openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) + coponentInfo.OpenEulerScore = openEulerScoreSlice + cveNumSlice := make([]string, 0) + cveNumSlice = append(cveNumSlice, v.CveNum) + branchCve := make(map[string][]string) + branchCve[affectBranch] = cveNumSlice + //cveNum := make([]map[string][]string, 0) + //cveNum = append(cveNum, branchCve) + coponentInfo.CveNumMap = branchCve + coponentInfo.CveNum = cveNumSlice + componentMap[influenceComponent] = coponentInfo + } else { + packNameStruct.UpdateFlag = 2 + repFlagX := false + if len(packNameStruct.CveNum) > 0 { + for _, cveX := range packNameStruct.CveNum { + if cveX == v.CveNum { + repFlagX = true + break + } + } + if !repFlagX { + packNameStruct.CveNum = append(packNameStruct.CveNum, v.CveNum) + packNameStruct.OpenEulerScore = append(packNameStruct.OpenEulerScore, v.OpenEulerScore) + } + } else { + cveNumSlice := make([]string, 0) + cveNumSlice = append(cveNumSlice, v.CveNum) + packNameStruct.CveNum = cveNumSlice + openEulerScoreSlice := make([]float64, 0) + openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) + packNameStruct.OpenEulerScore = openEulerScoreSlice + } + if len(packNameStruct.CveNumMap) > 0 { + if branchCve, brOk := packNameStruct.CveNumMap[affectBranch]; !brOk { + cveNumSlice := make([]string, 0) + cveNumSlice = append(cveNumSlice, v.CveNum) + packNameStruct.CveNumMap[affectBranch] = cveNumSlice + } else { + for _, cve := range branchCve { + if cve == v.CveNum { + repFlag = true + break + } + } + if !repFlag { + branchCve = append(branchCve, v.CveNum) + packNameStruct.CveNumMap[affectBranch] = branchCve + } + } + } else { + branchCve := make(map[string][]string) + cveNumSlice := make([]string, 0) + cveNumSlice = append(cveNumSlice, v.CveNum) + branchCve[affectBranch] = cveNumSlice + packNameStruct.CveNumMap = branchCve + } + componentMap[influenceComponent] = packNameStruct + } + return repFlag +} + +func saveCvrfName(cvrfFileList map[string][]string, cvrffileName, mapKey string) { + cvrfFileSlice, cvrfOk := cvrfFileList[mapKey] + if cvrfOk { + fileExist := false + for _, cvrfVuale := range cvrfFileSlice { + if cvrfVuale == cvrffileName { + fileExist = true + } + } + if !fileExist { + cvrfFileSlice = append(cvrfFileSlice, cvrffileName) + cvrfFileList[mapKey] = cvrfFileSlice + } + } else { + cvrfNameSlice := make([]string, 0) + cvrfNameSlice = append(cvrfNameSlice, cvrffileName) + cvrfFileList[mapKey] = cvrfNameSlice + } +} + +func procCvrfData(v models.ExcelExport, + affectBranch string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, + cvfrFileMap map[string]CvrfSa, pkg []models.Package) { + fileDir := beego.AppConfig.String("fileDir") + cvrffileName := filepath.Join(fileDir, "cvrf-"+componentMap[v.InfluenceComponent].OpenEulerSANum+".xml") + cvrfSaStruct, cvrfSaOk := cvfrFileMap[cvrffileName] + if !cvrfSaOk { + // Query whether there is data in the database + var cfr models.CvrfSaRecord + var cvrfsa CvrfSa + cfr.OpenEulerSANum = componentMap[v.InfluenceComponent].OpenEulerSANum + tbErr := models.GetCvrfRecord(&cfr, "openeuler_sa_num") + if tbErr == nil && cfr.Id > 0 { + // Download data from file server + + // Read file content + readErr := ReadCvrfXml(cvrffileName, &cvrfsa) + if readErr != nil { + cvrfsa = CvrfSa{} + } else { + var componentInfo ComponentInfo + componentInfo = componentMap[v.OwnedComponent] + componentInfo.UpdateFlag = 1 + componentMap[v.OwnedComponent] = componentInfo + } + } + BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg) + cvfrFileMap[cvrffileName] = cvrfsa + } else { + BuildCvrfXml(&cvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg) + cvfrFileMap[cvrffileName] = cvrfSaStruct + } + saveCvrfName(cvrfFileList, cvrffileName, CVRFFKEY) +} + +func (ec *CveExcel) setContentRow(v models.ExcelExport, + affectBranch string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { pkg, pkgErr := models.GetCvePackageList(v.SecID) if pkgErr != nil { logs.Error(pkgErr) @@ -628,70 +814,74 @@ func (ec *CveExcel) setContentRow(v models.ExcelExport, affectBranch string) { v.PublicDate = time.Now().Format("2006-01-02") } pkgStr := getPkgStr(pkg) - sn := []interface{}{v.OpenEulerSANum, v.CveNum, v.Introduction, v.Summary, v.Theme, v.Description, v.InfluenceComponent, - v.AffectProduct, pkgStr, v.ReferenceLink, v.PublicDate} - axis, searched := ec.searchValueInSheet(ec.SecNoticeSheetName, v.InfluenceComponent) - if !searched { - fillErr := ec.fillSecurityNoticeSheet(sn) - if fillErr != nil { - logs.Error(fillErr) - } - } else { - //merge openEuler SA notice data - colReg := regexp.MustCompile(`[A-Z]*`) - col := colReg.FindString(axis) - row := strings.Trim(axis, col) - rCN := "B" + row - rRl := "J" + row - vcn, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rCN) - vcn += ";\n" + v.CveNum - _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rCN, vcn) - vrl, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rRl) - vrl += "\n" + v.ReferenceLink - _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rRl, vrl) - rSAN := fmt.Sprintf("A%s", row) - vSAN, cellError := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rSAN) - if cellError == nil { - v.OpenEulerSANum = vSAN - } - //merger description - rd := "F" + row - vd, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rd) - dSplit := strings.Split(v.Description, "Security Fix(es):") - if len(dSplit) > 1 { - if !strings.Contains(vd, dSplit[0]) { - vd = dSplit[0] + vd + repFlag := StoreComponentInfo(componentMap, v, affectBranch) + if !repFlag { + procCvrfData(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap, pkg) + sn := []interface{}{v.OpenEulerSANum, v.CveNum, v.Introduction, v.Summary, v.Theme, v.Description, v.InfluenceComponent, + v.AffectProduct, pkgStr, v.ReferenceLink, v.PublicDate} + axis, searched := ec.searchValueInSheet(ec.SecNoticeSheetName, v.InfluenceComponent) + if !searched { + fillErr := ec.fillSecurityNoticeSheet(sn) + if fillErr != nil { + logs.Error(fillErr) + } + } else { + //merge openEuler SA notice data + colReg := regexp.MustCompile(`[A-Z]*`) + col := colReg.FindString(axis) + row := strings.Trim(axis, col) + rCN := "B" + row + rRl := "J" + row + vcn, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rCN) + vcn += ";\n" + v.CveNum + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rCN, vcn) + vrl, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rRl) + vrl += "\n" + v.ReferenceLink + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rRl, vrl) + rSAN := fmt.Sprintf("A%s", row) + vSAN, cellError := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rSAN) + if cellError == nil { + v.OpenEulerSANum = vSAN + } + //merger description + rd := "F" + row + vd, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rd) + dSplit := strings.Split(v.Description, "Security Fix(es):") + if len(dSplit) > 1 { + if !strings.Contains(vd, dSplit[0]) { + vd = dSplit[0] + vd + } + vd += dSplit[1] + } + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rd, vd) + //Get the highest-rated theme + rd = "E" + row + theme, err := models.GetCanExportTheme(vcn, v.OwnedComponent, affectBranch) + if err == nil && len(theme) > 1 { + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rd, theme) } - vd += dSplit[1] } - _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rd, vd) - //Get the highest-rated theme - rd = "E" + row - theme, err := models.GetCanExportTheme(vcn, v.OwnedComponent, affectBranch) - if err == nil && len(theme) > 1 { - _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rd, theme) + cve := []interface{}{v.CveNum, v.CveBrief, v.NVDScore, v.OpenEulerScore, v.NattackVector, v.OattackVector, + v.NattackComplexity, v.OattackComplexity, v.NprivilegeRequired, v.OprivilegeRequired, v.NuserInteraction, + v.OuserInteraction, v.Nscope, v.Oscope, v.Nconfidentiality, v.Oconfidentiality, v.Nintegrity, v.Ointegrity, + v.Navailability, v.Oavailability, v.ScoreType, v.OpenEulerSANum, v.PublicDate} + shErr := ec.fillCveSheetRow(cve) + if shErr != nil { + logs.Error(shErr) } - } - cve := []interface{}{v.CveNum, v.CveBrief, v.NVDScore, v.OpenEulerScore, v.NattackVector, v.OattackVector, - v.NattackComplexity, v.OattackComplexity, v.NprivilegeRequired, v.OprivilegeRequired, v.NuserInteraction, - v.OuserInteraction, v.Nscope, v.Oscope, v.Nconfidentiality, v.Oconfidentiality, v.Nintegrity, v.Ointegrity, - v.Navailability, v.Oavailability, v.ScoreType, v.OpenEulerSANum, v.PublicDate} - shErr := ec.fillCveSheetRow(cve) - if shErr != nil { - logs.Error(shErr) - } - ap := []interface{}{v.CveNum, v.AffectProduct, v.InfluenceComponent, v.AffectStatus} - err := ec.fillAffectProductSheet(ap) - if err != nil { - logs.Error(err) - } - for _, v := range pkg { - pk := []interface{}{v.PackName, v.PackUrl} - if _, ok := ec.searchValueInSheet(ec.PackageURLSheetName, v.PackName); !ok { - err := ec.fillPackageSheet(pk) - if err != nil { - logs.Error(err) + ap := []interface{}{v.CveNum, v.AffectProduct, v.InfluenceComponent, v.AffectStatus} + err := ec.fillAffectProductSheet(ap) + if err != nil { + logs.Error(err) + } + for _, v := range pkg { + pk := []interface{}{v.PackName, v.PackUrl} + if _, ok := ec.searchValueInSheet(ec.PackageURLSheetName, v.PackName); !ok { + err := ec.fillPackageSheet(pk) + if err != nil { + logs.Error(err) + } } } } @@ -828,6 +1018,7 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I // 查询当前需要处理的issue issueTemp, err := models.GetIssueNumber(v.Repo) if err != nil || issueTemp == nil { + logs.Info("No need to deal with ", v.Repo) continue } for _, isTemp := range issueTemp { @@ -839,7 +1030,8 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I repoIssue[p.Id] = p } if len(repoIssue) > 0 { - chData = append(chData, IssueAndPkg{IssueMap: repoIssue, IssuePkg: v.Packages, Repo: v.Repo}) + chData = append(chData, IssueAndPkg{IssueMap: repoIssue, IssuePkg: v.Packages, + Repo: v.Repo, IssueId: isTemp.IssueId}) } } } @@ -847,7 +1039,9 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I } func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, cvexml *[]CveXml, - dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) { + dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, + packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { defer wgTrigger.Done() data := <-c var pkgList []string @@ -859,24 +1053,27 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, continue } for _, iv := range v.IssueMap { - tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo} - err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo") + tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo, IssueId: v.IssueId} + err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo", "issue_id") + //tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo} + //err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo") if err != nil { - logs.Error("GetIssueTemplateByColName, ----", err) + logs.Error("GetIssueTemplateByColName, ----", err, iv.Number, iv.Repo, v.IssueId) continue } err = models.ReplacePackageByCveId(pkgList, tpl.CveId) if err != nil { - logs.Info("ReplacePackageByCveId, err: ", err) + logs.Error("ReplacePackageByCveId, err: ", err, "tpl.CveId: ", tpl.CveId) continue } //save data to excel el, err := models.GetCanExportExcelData(tpl.CveNum, tpl.IssueNum) if err != nil { - logs.Error("GetCanExportExcelData, err: ", err) + logs.Error("GetCanExportExcelData, err: ", err, "tpl.CveNum, tpl.IssueNum: ", tpl.CveNum, tpl.IssueNum) return } - err = ec.handleWriteContentSync(el, affectBranch, cvexml, dpdates, securityNotice, packRpmx) + err = ec.handleWriteContentSync(el, affectBranch, cvexml, dpdates, + securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap) if err != nil { logs.Error("handleWriteContentSync, err: ", err) } diff --git a/taskhandler/sendemail.go b/taskhandler/sendemail.go index e7de9538d10f2a4e08ba82e8d59683e30544f4dd..fb146d1c5b2156433096b0584278dff30acdf1ee 100644 --- a/taskhandler/sendemail.go +++ b/taskhandler/sendemail.go @@ -3,14 +3,18 @@ package taskhandler import ( "archive/zip" "bytes" + "cvevulner/common" "cvevulner/models" "encoding/base64" + "fmt" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" + "gopkg.in/gomail.v2" "io" "io/ioutil" "net/smtp" "os" + "strconv" "strings" "time" ) @@ -56,14 +60,12 @@ func ZipFiles(filename string, files []string, oldform, newform string) error { defer newZipFile.Close() zipWriter := zip.NewWriter(newZipFile) defer zipWriter.Close() - // 把files添加到zip中 for _, file := range files { zipfile, err := os.Open(file) if err != nil { return err } defer zipfile.Close() - // 获取file的基础信息 info, err := zipfile.Stat() if err != nil { return err @@ -72,10 +74,7 @@ func ZipFiles(filename string, files []string, oldform, newform string) error { if err != nil { return err } - //使用上面的FileInforHeader() 就可以把文件保存的路径替换成我们自己想要的了，如下面 header.Name = strings.Replace(file, oldform, newform, -1) - // 优化压缩 - // 更多参考see http://golang.org/pkg/archive/zip/#pkg-constants header.Method = zip.Deflate writer, err := zipWriter.CreateHeader(header) if err != nil { @@ -88,8 +87,7 @@ func ZipFiles(filename string, files []string, oldform, newform string) error { return nil } - -func SendEmail(attchStr string) error { +func SendEmail(attchStr string, flag int) error { var mail Mail emailName := beego.AppConfig.String("email::email_name") emailPwd := beego.AppConfig.String("email::email_pwd") @@ -105,21 +103,50 @@ func SendEmail(attchStr string) error { for _, em := range el { toEmailName = append(toEmailName, em.EmailName) } + //_, attchName := filepath.Split(attchStr) + emailError := error(nil) mail = &SendMail{user: emailName, password: emailPwd, host: emailHost, port: emailPort} - message := Message{from: emailName, - to: toEmailName, - cc: []string{}, - bcc: []string{}, - subject: "updateinfo.xml and the excel file corresponding to SA", - body: "hi all: \n The attachment is: updateinfo.xml and the excel file corresponding to SA.", - contentType: "text/plain;charset=utf-8", - attachment: Attachment{ - name: attchStr, - contentType: "text/plain", - withFile: true, - }, + if flag == 1 { + message := Message{from: emailName, + to: toEmailName, + cc: []string{}, + bcc: []string{}, + subject: "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime(), + body: fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been uploaded to the file server is as follows: \r\n" + attchStr), + contentType: "text/plain;charset=utf-8", + attachment: Attachment{ + name: attchStr, + contentType: "text/plain", + withFile: true, + }, + } + emailError = mail.Send(message) + if emailError == nil { + logs.Info("Notify cvrf that the email was sent successfully!") + } else { + logs.Error("Notify cvrf mail delivery failure!") + } + } else { + message := Message{from: emailName, + to: toEmailName, + cc: []string{}, + bcc: []string{}, + subject: "updateinfo.xml and the excel file corresponding to SA", + body: "hi all: \r\n The attachment is: updateinfo.xml and the excel file corresponding to SA.", + contentType: "text/plain;charset=utf-8", + attachment: Attachment{ + name: attchStr, + contentType: "text/plain", + withFile: true, + }, + } + emailError = mail.Send(message) + if emailError == nil { + logs.Info("updateinfo.xml and the excel file corresponding to SA, Mail sent successfully!") + } else { + logs.Error("updateinfo.xml and the excel file corresponding to SA, Mail sent failure!") + } } - emailError := mail.Send(message) return emailError } @@ -132,7 +159,8 @@ func (mail SendMail) Send(message Message) error { buffer := bytes.NewBuffer(nil) boundary := "GoBoundary" Header := make(map[string]string) - Header["From"] = message.from + //Header["From"] = message.from + Header["From"] = "cve-manager" + "<" + message.from + ">" Header["To"] = strings.Join(message.to, ";") Header["Cc"] = strings.Join(message.cc, ";") Header["Bcc"] = strings.Join(message.bcc, ";") @@ -189,4 +217,47 @@ func (mail SendMail) writeFile(buffer *bytes.Buffer, fileName string) { buffer.WriteString("\r\n") } } -} \ No newline at end of file +} + +func SendCommonMail(mailTo []string, subject string, body string) error { + emailName := beego.AppConfig.String("email::email_name") + emailPwd := beego.AppConfig.String("email::email_pwd") + emailHost := beego.AppConfig.String("email::email_host") + //emailPort := beego.AppConfig.String("email::email_port") + mailConn := map[string]string{ + "user": emailName, + "pass": emailPwd, + "host": emailHost, + "port": "465", + } + port, _ := strconv.Atoi(mailConn["port"]) + m := gomail.NewMessage() + m.SetHeader("From", "cve-manager"+"<"+mailConn["user"]+">") + m.SetHeader("To", mailTo...) + m.SetHeader("Subject", subject) + m.SetBody("text/plain", body) + d := gomail.NewDialer(mailConn["host"], port, mailConn["user"], mailConn["pass"]) + err := d.DialAndSend(m) + return err +} + +func SendCvrfEmail(subFileSlice []string) { + elt := models.EmailList{EmailType: 1} + el, eltErr := elt.Read("EmailType") + if eltErr != nil { + logs.Error("Failed to get mailing list, err: ", eltErr) + } + toEmailName := []string{} + for _, em := range el { + toEmailName = append(toEmailName, em.EmailName) + } + subject := "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime() + fileStr := strings.Join(subFileSlice, "\r\n") + body := fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been uploaded to the file server is as follows: \r\n" + fileStr) + sendErr := SendCommonMail(toEmailName, subject, body) + if sendErr == nil { + logs.Info("Notify cvrf that the email was sent successfully!") + } else { + logs.Error("Notify cvrf mail delivery failure!") + } +} diff --git a/taskhandler/xml.go b/taskhandler/xml.go index 69cde7055c780c5365ed251c6ed167957e9c2de2..6e7186658bccd16670b467dc74fbb14707bf27a6 100644 --- a/taskhandler/xml.go +++ b/taskhandler/xml.go @@ -3,10 +3,10 @@ package taskhandler import ( "cvevulner/models" "encoding/xml" - "fmt" "github.com/astaxie/beego/logs" "io/ioutil" "os" + "sort" "strings" "time" ) @@ -71,11 +71,12 @@ type Package struct { // Intermediate information type SecurityNoticeXml struct { - CveNum string - Title string - Type string - ReferenceLink string - Description string + CveNum string + Title string + Type string + ReferenceLink string + Description string + openEulerScore float64 } type PackRpm struct { @@ -96,6 +97,11 @@ func ReadXml(filePath string, dpdates *Updates) error { if filePath == "" || len(filePath) == 0 { filePath = "download/updateinfo.xml" } + fisExist, ferr := PathExists(filePath) + if !fisExist { + logs.Error(ferr) + return ferr + } fd, err := os.Open(filePath) if err != nil { logs.Error("open file err : ", err, ",filePath: ", filePath) @@ -126,7 +132,8 @@ func WriteXml(filePath string, cveXmlList []CveXml, dpdates *Updates, var rfs References if sn, ok := securityNotice[cveXml.OwnedComponent]; ok { if len(sn) > 0 { - descriptionsList := []string{} + descriptionsList := make([]string, 0) + openEulerScoreSlice := make([]float64, 0) for _, sec := range sn { var rf Reference rf.Id = sec.CveNum @@ -134,18 +141,35 @@ func WriteXml(filePath string, cveXmlList []CveXml, dpdates *Updates, rf.Href = sec.ReferenceLink rf.Type = "cve" descriptionsList = append(descriptionsList, sec.Description) + openEulerScoreSlice = append(openEulerScoreSlice, sec.openEulerScore) rfs.Reference = append(rfs.Reference, rf) } upDatex.Description = strings.Join(descriptionsList, " ") + if len(openEulerScoreSlice) > 1 { + sort.Float64s(openEulerScoreSlice) + } + cveLevel := openEulerScoreProc(openEulerScoreSlice[len(openEulerScoreSlice)-1]) + if strings.ToLower(cveLevel) == "low" { + upDatex.Severity = "Low" + } else if strings.ToLower(cveLevel) == "medium" { + upDatex.Severity = "Moderate" + } else if strings.ToLower(cveLevel) == "high" { + upDatex.Severity = "Important" + } else { + upDatex.Severity = "Critical" + } } } upDatex.References = &rfs var pl Pkglist var ct Collection ct.Name = "openEuler" - if prx, ok := packRpmx[cveXml.OwnedComponent]; ok{ + if prx, ok := packRpmx[cveXml.OwnedComponent]; ok { if len(prx) > 0 { for _, pr := range prx { + if strings.Contains(pr.PackName, ".src.") { + continue + } var pe Package pe.Filename = pr.PackName //pe.Name = cveXml.OwnedComponent @@ -153,14 +177,14 @@ func WriteXml(filePath string, cveXmlList []CveXml, dpdates *Updates, //packVersion := pr.PackName[len(cveXml.OwnedComponent) + 1: len(pr.PackName) - 4] packVersionList := strings.Split(pr.PackName, "-") if len(packVersionList) >= 3 { - pe.Version = packVersionList[len(packVersionList) - 2] - rpmName := packVersionList[len(packVersionList) - 1][:len(packVersionList[len(packVersionList) - 1]) - 4] + pe.Version = packVersionList[len(packVersionList)-2] + rpmName := packVersionList[len(packVersionList)-1][:len(packVersionList[len(packVersionList)-1])-4] lastIndex := strings.LastIndexAny(rpmName, ".") if lastIndex != -1 { pe.Release = rpmName[:lastIndex] - pe.Arch = rpmName[lastIndex + 1:] + pe.Arch = rpmName[lastIndex+1:] } - pe.Name = strings.Join(packVersionList[0:len(packVersionList) - 2], "-") + pe.Name = strings.Join(packVersionList[0:len(packVersionList)-2], "-") } } ct.Package = append(ct.Package, pe) @@ -171,16 +195,14 @@ func WriteXml(filePath string, cveXmlList []CveXml, dpdates *Updates, upDatex.Pkglist = &pl dpdates.Updatex = append(dpdates.Updatex, upDatex) } - xmlOutPut, outPutErr := xml.MarshalIndent(dpdates, "", " ") + xmlOutPut, outPutErr := xml.MarshalIndent(dpdates, "", " ") if outPutErr == nil { - //加入XML头 headerBytes := []byte(xml.Header) - //拼接XML头和实际XML内容 xmlOutPutData := append(headerBytes, xmlOutPut...) - //写入文件 + os.Remove(filePath) ioutil.WriteFile(filePath, xmlOutPutData, os.ModeAppend) } else { - fmt.Println(outPutErr) + logs.Error(outPutErr) } } @@ -193,7 +215,7 @@ func BuildXml(cveXml *[]CveXml, v *models.ExcelExport, } //cvex.SecurityNotice = make(map[string][]SecurityNoticeXml) sn := SecurityNoticeXml{CveNum: v.CveNum, Title: v.CveNum, Type: "cve", ReferenceLink: v.ReferenceLink, - Description: deleteTailBlank(v.CveBrief) + "(" + v.CveNum + ")"} + Description: deleteTailBlank(v.CveBrief) + "(" + v.CveNum + ")", openEulerScore: v.OpenEulerScore} if vx, ok := securityNotice[v.OwnedComponent]; !ok { snArry := []SecurityNoticeXml{} snArry = append(snArry, sn) @@ -213,7 +235,11 @@ func BuildXml(cveXml *[]CveXml, v *models.ExcelExport, } else { cvex.CveLevel = "Critical" } - cvex.Introduction = v.Introduction + if len(v.Introduction) > 1 { + cvex.Introduction = v.Introduction[:len(v.Introduction)-1] + } else { + cvex.Introduction = v.Introduction + } cvex.OwnedComponent = v.OwnedComponent *cveXml = append(*cveXml, cvex) } else {