From 9509d2573f2016ff4d1e634b923e24d030bbddec Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Thu, 4 Feb 2021 14:30:25 +0800 Subject: [PATCH 1/2] Solve the cvrf format data error generated by multiple branches --- conf/product_app.conf | 2 +- controllers/file.go | 7 ++++--- controllers/hook.go | 26 ++++++++++++++++++++++++++ models/giteeissue.go | 7 +++++++ models/modeldb.go | 7 ++++--- taskhandler/cve.go | 2 +- taskhandler/cvrf.go | 8 +++++--- taskhandler/excel.go | 34 ++++++++++++++++++++-------------- taskhandler/issuestatistics.go | 17 +++++++++-------- 9 files changed, 77 insertions(+), 33 deletions(-) diff --git a/conf/product_app.conf b/conf/product_app.conf index 6d95624..c3ab873 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -145,7 +145,7 @@ snsuffix = 1002 # Version package excel download address # example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; # openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv -v_pack_20_03_url = "openEuler-20.03-LTS@https://openEuler-20.03-LTS.csv;openEuler-20.03-LTS-SP1@https://openEuler-20.03-LTS-SP1.csv" +v_pack_20_03_url = "openEuler-20.03-LTS@https;openEuler-20.03-LTS-SP1@https" # Time difference in different time zones sa_timestamp_zone = 28800 diff --git a/controllers/file.go b/controllers/file.go index b4b9ea0..8ed8c2c 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -124,7 +124,6 @@ func (f *FileController) TriggerCveData() { csvPathList := strings.Split(du, ";") if len(csvPathList) > 0 { for _, csvP := range csvPathList { - time.Sleep(time.Second) openBranchx := strings.Split(csvP, "@") if len(openBranchx) == 2 { affectBranch := openBranchx[0] @@ -233,10 +232,11 @@ func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, } } cvrfFileList[taskhandler.CVRFFKEY] = writeCvrfSlice - uploadCvrfFile(cvrfFileList, totalFileSlice, dir) + uploadCvrfFile(cvrfFileList, totalFileSlice, dir, componentMap) } -func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, dir string) { +func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, dir string, + componentMap map[string]taskhandler.ComponentInfo) { uploadPath := beego.AppConfig.String("xml::cvrf_upload_path") // File name and data stored in database fileSlice, fOk := cvrfFileList[taskhandler.CVRFFKEY] @@ -297,6 +297,7 @@ func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, d subFileSlice := make([]string, 0) updateFilePath := filepath.Join(dir, "update_unaffect.txt") for _, fPath := range unaffFileSlice { + taskhandler.WriteUnaffectCvrfXml(fPath, componentMap[fPath].UnaffectFile) _, fileName := filepath.Split(fPath) // File storage to db recordErr := taskhandler.RecordCrvfInfo(fPath, fileName, taskhandler.UNAFFECTFLAG) diff --git a/controllers/hook.go b/controllers/hook.go index ec5359a..71e0cce 100644 --- a/controllers/hook.go +++ b/controllers/hook.go @@ -2100,6 +2100,32 @@ func DelOrgIssue(issueHook *models.IssuePayload) { if err != nil { logs.Error("Failed to delete issue:", err) } + issueTmp := models.IssueTemplate{} + issueTmp.IssueId = issueHook.Issue.Id + issueTmp.IssueNum = issueNumber + issueTmp.OwnedComponent = repoPath + issueErr := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "owned_component", "issue_id") + if issueErr != nil { + return + } + issueTmp.Status = 6 + issueTmp.IssueStatus = 6 + issueTmp.StatusName = "已删除" + tpErr := models.UpdateIssueTemplate(&issueTmp, "status", "issue_status", + "status_name") + if tpErr != nil { + logs.Error(tpErr) + } + cveCenter := models.VulnCenter{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum, PackName: issueTmp.OwnedComponent} + cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num", "pack_name") + if cveErr != nil { + return + } + cveCenter.Status = 7 + update := models.UpdateVulnCenter(&cveCenter, "cve_status") + if !update { + logs.Error("update vulnCenter fail ") + } } } diff --git a/models/giteeissue.go b/models/giteeissue.go index 69d49bc..2cb6887 100644 --- a/models/giteeissue.go +++ b/models/giteeissue.go @@ -106,6 +106,13 @@ func (g *GiteOriginIssue) Detlete() error { return err } +func UpdateCveCenterStatus(cveId int64, cveNum string, status int8) { + o := orm.NewOrm() + _ = o.Raw("UPDATE cve_vuln_center SET cve_status = ? WHERE cve_id = ? and cve_num = ?", + status, cveId, cveNum).QueryRow() + return +} + func GetGiteOriginIssue(cveNum string) (oi GiteOriginIssue, err error) { o := orm.NewOrm() oi.CveNumber = cveNum diff --git a/models/modeldb.go b/models/modeldb.go index 763e55c..d51c3d7 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -38,7 +38,8 @@ type VulnCenter struct { CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` - Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求);5:issue已经创建过，不符合要求，不处理;6:字段为空(评分,描述等)"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求); + 5:issue已经创建过，不符合要求，不处理;6:字段为空(评分,描述等);7:issue已被删除不再创建"` CveVersion string `orm:"size(512);column(cve_version)" description:"cve归属版本,版本合并"` RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` @@ -151,9 +152,9 @@ type IssueTemplate struct { IssueId int64 `orm:"column(issue_id)" description:"issue的id"` IssueNum string `orm:"size(64);column(issue_num);index" description:"issue编号"` Assignee string `orm:"size(128);column(issue_assignee)" description:"issue所属责任人"` - Status int8 `orm:"default(1);column(status)" description:"1:待办的；2：进行中；3：已完成；4：已拒绝；5： 已挂起"` + Status int8 `orm:"default(1);column(status)" description:"1:待办的；2：进行中；3：已完成；4：已拒绝；5： 已挂起; 6:已删除"` StatusName string `orm:"size(128);column(status_name)" description:"issue状态名称"` - IssueStatus int8 `orm:"default(1);column(issue_status)" description:"1:待分析；2：已正常关闭；3已分析，待修复；4：已修复；5：已发布；6：已异常关闭"` + IssueStatus int8 `orm:"default(1);column(issue_status)" description:"1:待分析；2：已正常关闭；3：已分析，待修复；4：已修复；5：已发布；6：已异常关闭"` IssueLabel string `orm:"size(256);column(issue_label)" description:"issue标签, CVE/FIXED, CVE/UNFIXED"` MtAuditFlag int8 `orm:"default(0);column(mt_audit_flag)" description:"maintainer 审核： 0：审核中；1：审核通过；2：审核拒绝"` SaAuditFlag int8 `orm:"default(0);column(sa_audit_flag)" description:"pr审核： 0：审核中；1：审核通过；2：审核拒绝"` diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 4a39365..36ac0f2 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -2055,7 +2055,7 @@ func GiteeDownCsv(localPath, branch string) (error) { logs.Error(err) return err } - logs.Info("body: \n", string(body), "url: ", gitUrl) + //logs.Info("body: \n", string(body), "url: ", gitUrl) var contents map[string]interface{} err = json.Unmarshal(body, &contents) if err != nil { diff --git a/taskhandler/cvrf.go b/taskhandler/cvrf.go index 5a8e21f..f861cdf 100644 --- a/taskhandler/cvrf.go +++ b/taskhandler/cvrf.go @@ -271,6 +271,7 @@ type ComponentInfo struct { OpenEulerScore []float64 UpdateFlag int CvrfFileName string + UnaffectFile *UnaffectCvrfSa } type UnaffectCvrfSa struct { @@ -334,8 +335,8 @@ func WriteUnaffectCvrfXml(filePath string, unaffectCvrfsa *UnaffectCvrfSa) { func BuildUnaffectCvrfXml(unaffectCvrfsa *UnaffectCvrfSa, v models.ExcelExport, affectBranch string, componentMap map[string]ComponentInfo) { - v.Description = strings.ReplaceAll(v.Description, "\n\n", " ") - v.Theme = strings.ReplaceAll(v.Theme, "\n\n", " ") + //v.Description = strings.ReplaceAll(v.Description, "\n\n", " ") + //v.Theme = strings.ReplaceAll(v.Theme, "\n\n", " ") unaffectCvrfsa.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" unaffectCvrfsa.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" BuildUnaffectVulnerabilitySet(unaffectCvrfsa, v, affectBranch, componentMap) @@ -402,7 +403,7 @@ func BuildUnaffVulnerabilitySlice(vulnerability []UnaffectVulnerability, v model cpe := affectBranch if vulnerability != nil && len(vulnerability) > 0 { cveExist := false - for _, vl := range vulnerability { + for i, vl := range vulnerability { if vl.Cve == v.CveNum && vl.Remediations != nil && len(vl.Remediations.Remediation) > 0 && vl.Remediations.Remediation[0].Description == v.InfluenceComponent { cpeExist := false @@ -422,6 +423,7 @@ func BuildUnaffVulnerabilitySlice(vulnerability []UnaffectVulnerability, v model remediation.Date = common.GetCurDate() remediation.ProductId = cpe vl.Remediations.Remediation = append(vl.Remediations.Remediation, remediation) + vulnerability[i] = vl } cveExist = true break diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 053c20c..872d74f 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -527,13 +527,22 @@ func procUnaffectCvrfData(v models.ExcelExport, cvrfFileList[UNAFFECTCVRFKEY] = cvrfNameSlice } var unaffectcvrf UnaffectCvrfSa - // Read file content - readErr := UnaffectReadCvrfXml(cvrffileName, &unaffectcvrf) - if readErr != nil { - unaffectcvrf = UnaffectCvrfSa{} + fileValue, ok := componentMap[cvrffileName] + if ok { + if len(fileValue.UnaffectFile.Vulnerability) > 0 { + unaffectcvrf = *componentMap[cvrffileName].UnaffectFile + } } + // Read file content + //readErr := UnaffectReadCvrfXml(cvrffileName, &unaffectcvrf) + //if readErr != nil { + // unaffectcvrf = UnaffectCvrfSa{} + //} BuildUnaffectCvrfXml(&unaffectcvrf, v, affectBranch, componentMap) - WriteUnaffectCvrfXml(cvrffileName, &unaffectcvrf) + var cif ComponentInfo + cif.UnaffectFile = &unaffectcvrf + componentMap[cvrffileName] = cif + //WriteUnaffectCvrfXml(cvrffileName, &unaffectcvrf) //saveCvrfName(cvrfFileList, cvrffileName, UNAFFECTCVRFKEY) } @@ -545,7 +554,7 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, lz := len(list) if lz > 0 { for _, v := range list { - logs.Info("handleWriteContentSync, v====>", v) + //logs.Info("handleWriteContentSync, v====>", v) if v.OrganizateId == 2 { logs.Error("opengauss, data: ", v) continue @@ -879,10 +888,10 @@ func BranchCvrfData(v models.ExcelExport, } BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) cvfrFileMap[cvrffileName] = cvrfsa - logs.Info("xxxx1====>", cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg) + //logs.Info("xxxx1====>", cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg) } else { BuildCvrfXml(&branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) - logs.Info("xxxx2====>", branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg) + //logs.Info("xxxx2====>", branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg) cvfrFileMap[cvrffileName] = branchCvrfSaStruct } saveCvrfName(cvrfFileList, cvrffileName, CVRFFKEY) @@ -1111,7 +1120,7 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I st := util.TimeStrToInt(startTime, "2006-01-02") chData := make([]IssueAndPkg, 0) for _, v := range pkgList { - logs.Info("excel, v===>", v) + //logs.Info("excel, v===>", v) rt := util.TimeStrToInt(v.PubTime, "20060102 15-04-05") + saTimeStampZone // 查询当前需要处理的issue issueTemp, err := models.GetIssueNumber(v.Repo) @@ -1144,7 +1153,7 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, data := <-c var pkgList []string for _, v := range data { - logs.Info("v====>", v) + //logs.Info("v====>", v) //parse package string to list pkgList = strings.Split(v.IssuePkg, " ") if len(pkgList) == 0 { @@ -1154,8 +1163,6 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, for _, iv := range v.IssueMap { tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo, IssueId: v.IssueId} err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo", "issue_id") - //tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo} - //err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo") if err != nil { logs.Error("GetIssueTemplateByColName, ----", err, iv.Number, iv.Repo, v.IssueId) continue @@ -1221,7 +1228,7 @@ func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, mt := util.TimeStrToInt(closedAt[:19], "2006-01-02T15:04:05") mergedAt := v["merged_at"].(string) ct := util.TimeStrToInt(mergedAt[:19], "2006-01-02T15:04:05") - logs.Info("******, pr: ", v, ",mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct, ",repo:", repo) + logs.Info("******>>mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct, ",repo:", repo, ", pr: ", v) //ct := v["merged_at"].(string).(time.Time).Local().Unix() var pt int64 if mt > 0 && ct > 0 { @@ -1242,7 +1249,6 @@ func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, pr.CveNumber = isTemp.CveNum pr.Repo = repo prList = append(prList, pr) - logs.Info("prList====>", prList) } } } diff --git a/taskhandler/issuestatistics.go b/taskhandler/issuestatistics.go index 6758728..25576ca 100644 --- a/taskhandler/issuestatistics.go +++ b/taskhandler/issuestatistics.go @@ -223,15 +223,16 @@ func IssueStatistics(beforeDate, prcnum int, owner string) error { break } } - //zipFileName := pressFileZip(excelPath, dir) zipFileName := excelPath - if templateId > 0 { - cBody := fmt.Sprintf("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, 10) + "个, 其中" + - strconv.FormatInt(is.HighCveCount, 10) + "个7分以上漏洞, 详情见附件, 请在20号之前解决, 优先解决7分以上CVE. \r\n" + - "已经分析完毕的issue请maintainer尽快关掉, 否则影响数据统计; 提交PR时要关联issue, 若CVE在之前PR解决, 请PR提交人编辑PR信息将issue关联上. \r\n") - sendError := SendEmail(zipFileName, 2, cBody, "") - if sendError != nil { - logs.Error("SendEmail, sendErr: ", sendError) + if is.cveCount > 0 { + if templateId > 0 { + cBody := fmt.Sprintf("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, 10) + "个, 其中" + + strconv.FormatInt(is.HighCveCount, 10) + "个7分以上漏洞, 详情见附件, 请在20号之前解决, 优先解决7分以上CVE. \r\n" + + "已经分析完毕的issue请maintainer尽快关掉, 否则影响数据统计; 提交PR时要关联issue, 若CVE在之前PR解决, 请PR提交人编辑PR信息将issue关联上. \r\n") + sendError := SendEmail(zipFileName, 2, cBody, "") + if sendError != nil { + logs.Error("SendEmail, sendErr: ", sendError) + } } } fileSlice = append(fileSlice, excelPath) -- Gitee From aff7c6339357ba6fd3e2c887266be1b703997ccd Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Thu, 4 Feb 2021 14:46:11 +0800 Subject: [PATCH 2/2] Solve the cvrf format data error generated by multiple branches --- taskhandler/excel.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 872d74f..d79be6b 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -55,7 +55,7 @@ type IssueAndPkg struct { var fillLock sync.Mutex var wgTrigger sync.WaitGroup - +var cveMap sync.Mutex //var cvrfLock sync.Mutex //GenerateCveExcel Generate Excel documents based on data. @@ -918,7 +918,9 @@ func (ec *CveExcel) setContentRow(v models.ExcelExport, v.PublicDate = time.Now().Format("2006-01-02") } pkgStr := getPkgStr(pkg) + cveMap.Lock() repFlag := StoreComponentInfo(componentMap, v, affectBranch) + cveMap.Unlock() if !repFlag { procCvrfData(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap, pkg) sn := []interface{}{v.OpenEulerSANum, v.CveNum, v.Introduction, v.Summary, v.Theme, v.Description, v.InfluenceComponent, @@ -1228,7 +1230,7 @@ func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, mt := util.TimeStrToInt(closedAt[:19], "2006-01-02T15:04:05") mergedAt := v["merged_at"].(string) ct := util.TimeStrToInt(mergedAt[:19], "2006-01-02T15:04:05") - logs.Info("******>>mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct, ",repo:", repo, ", pr: ", v) + //logs.Info("******>>mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct, ",repo:", repo, ", pr: ", v) //ct := v["merged_at"].(string).(time.Time).Local().Unix() var pt int64 if mt > 0 && ct > 0 { -- Gitee