diff --git a/cve-vulner-manager/common/analysis.go b/cve-vulner-manager/common/analysis.go index 5d360952f8eaeeaf8a3edeeb8cb35fd864b32729..07ba5cccc562ccaab75d1705e0de7ce07d893841 100644 --- a/cve-vulner-manager/common/analysis.go +++ b/cve-vulner-manager/common/analysis.go @@ -15,8 +15,9 @@ const ( AnalysisNotExecute = "不受影响-漏洞代码不在执行路径" AnalysisCodeNotPresent = "不受影响-漏洞代码不存在" - TypeAffected = "Affected" - TypeUnaffected = "Unaffected" + TypeAffected = "Affected" + TypeUnaffected = "Unaffected" + TypeUnderInvestigation = "UnderInvestigation" ) var AnalysisUnaffected = map[string]struct{}{ diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 6e6767a5560c4a7be2a294257b49ca95a37def78..c446f9afece8a9fe782c4f987d7677bd1413b972 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -132,7 +132,7 @@ syncissuedateflag = 1 syncissuedate = 0 0 7,13 * * * releaseUnaffectedCveflag = 1 -releaseUnaffectedCve = 0 0 11 * * 5 +releaseUnaffectedCve = 0 0 11 * * * [gitee] diff --git a/cve-vulner-manager/controllers/file.go b/cve-vulner-manager/controllers/file.go index 3b977d4ddf0693c96ad0230166408b928d33b81a..88e155353a2aa6a029ce4abc429ce09f4aaad803 100644 --- a/cve-vulner-manager/controllers/file.go +++ b/cve-vulner-manager/controllers/file.go @@ -83,6 +83,9 @@ func (f *FileController) DownLoadExcelByFileCode() { // TriggerCveData touch off generate cve data excel and get cve package // @router /triggerCveData [get] func (f *FileController) TriggerCveData() { + // 停止维护 + f.Ctx.WriteString("Deprecated api") + return // Limit on the number of triggers nameStr, limitCount := LimitTriggerSa() if limitCount != 0 { diff --git a/cve-vulner-manager/cve-ddd/controller/cve.go b/cve-vulner-manager/cve-ddd/controller/cve.go index 3fcc57a8affa2089730c2f5fc347e6b9383e3857..6a19efd2361c2c7303ff2e397334d4d744a455ae 100644 --- a/cve-vulner-manager/cve-ddd/controller/cve.go +++ b/cve-vulner-manager/cve-ddd/controller/cve.go @@ -62,16 +62,6 @@ func (c *CveController) CollectCveData() { } func (c *CveController) Generate() { - concurrency.Add(1) - defer concurrency.Add(-1) - - // 公告接口不允许并发访问 - if concurrency.Load() > allowConcurrency { - c.fail("job is running") - - return - } - var request GenerateRequest if err := json.Unmarshal(c.Ctx.Input.RequestBody, &request); err != nil { c.fail(err.Error()) @@ -87,15 +77,33 @@ func (c *CveController) Generate() { return } - uploadDir, err := c.BulletinService.GenerateBulletins(request.CveNum, request.Date) - if err != nil { - c.BulletinLog.Errorf("generate security bulletins failed: %s", err.Error()) - } else { - err = c.HotPatchService.GenerateBulletins(uploadDir) - if err != nil { - c.HotPatchBulletinLog.Errorf("generate hot patch security bulletins failed: %s", err.Error()) + go func() { + defer func() { + if r := recover(); r != nil { + c.BulletinLog.Errorf("handle collect panic: %v", r) + } + }() + + concurrency.Add(1) + defer concurrency.Add(-1) + + // 公告接口不允许并发访问 + if concurrency.Load() > allowConcurrency { + c.BulletinLog.Errorf("job is running") + + return } - } + + uploadDir, err1 := c.BulletinService.GenerateBulletins(request.CveNum, request.Date) + if err1 != nil { + c.BulletinLog.Errorf("generate security bulletins failed: %s", err1.Error()) + } else { + err1 = c.HotPatchService.GenerateBulletins(uploadDir) + if err1 != nil { + c.HotPatchBulletinLog.Errorf("generate hot patch security bulletins failed: %s", err1.Error()) + } + } + }() c.success(nil) } diff --git a/cve-vulner-manager/main.go b/cve-vulner-manager/main.go index 9bff947d6218dfb3aa7635f89eb53d524b3361f0..0c6a1b5fecc46ebc9bd5cde4ca38f8fa33bd0d07 100644 --- a/cve-vulner-manager/main.go +++ b/cve-vulner-manager/main.go @@ -1,8 +1,6 @@ package main import ( - "time" - "github.com/astaxie/beego" "cvevulner/common" @@ -37,7 +35,6 @@ func main() { } taskhandler.InitReleaseDate() - taskhandler.InitAssignerCache(time.Now().Format("20060102")) // Initialize a scheduled task taskOk := task.InitTask() diff --git a/cve-vulner-manager/models/cve.go b/cve-vulner-manager/models/cve.go index 53ba282cb64a9c3b04b2db101fe12fb8d6dd7440..1bee1d73ef48d961f621c6fdafd522e42c83653e 100644 --- a/cve-vulner-manager/models/cve.go +++ b/cve-vulner-manager/models/cve.go @@ -1110,7 +1110,8 @@ is_export in (0,3) and pack_name in ('%s') and organizate_id = 1) and status < 4 func GetUnffectIssueNumber(startTime string, cves []string) (issueTemp []IssueTemplate, err error) { var sql string if len(cves) == 0 { - sql = `SELECT * FROM cve_issue_template WHERE STATUS = 3 AND issue_status in (2,6) AND cve_id IN (SELECT DISTINCT cve_id FROM cve_vuln_center WHERE cve_status = 2 AND + sql = `SELECT * FROM cve_issue_template WHERE STATUS <= 3 AND issue_status in (1,2,3,6) AND cve_id IN ( +SELECT DISTINCT cve_id FROM cve_vuln_center WHERE cve_status = 2 AND is_export IN (0,3) and organizate_id = 1) AND create_time >= '%s'` } else { var s string @@ -1120,7 +1121,8 @@ is_export IN (0,3) and organizate_id = 1) AND create_time >= '%s'` if len(s) > 1 { s = s[:len(s)-1] } - sql = `SELECT * FROM cve_issue_template WHERE STATUS = 3 AND issue_status in (2,6) AND cve_id IN (SELECT DISTINCT cve_id FROM cve_vuln_center WHERE cve_status = 2 AND + sql = `SELECT * FROM cve_issue_template WHERE STATUS <= 3 AND issue_status in (1,2,3,6) AND cve_id IN ( +SELECT DISTINCT cve_id FROM cve_vuln_center WHERE cve_status = 2 AND is_export IN (0,3) and organizate_id = 1) AND create_time >= '%s' ` + ` AND cve_num in (` + s + `)` } diff --git a/cve-vulner-manager/models/cve_web.go b/cve-vulner-manager/models/cve_web.go index 2a3a0ab2cb037d562a9ccb79680706ba3e551a06..f873f29fe2302e97351f8a0e1dac051e6005a254 100644 --- a/cve-vulner-manager/models/cve_web.go +++ b/cve-vulner-manager/models/cve_web.go @@ -25,9 +25,15 @@ type RespCveProduct struct { type CveProduct struct { Id int64 `json:"id"` + Status string `json:"status"` ProductName string `json:"productName"` } +// IsFixed checks if the status of the CveProduct is "Fixed". +func (c CveProduct) IsFixed() bool { + return c.Status == "Fixed" +} + type Cve struct { CveNum string `json:"cveNum"` Pack string `json:"packageName"` diff --git a/cve-vulner-manager/models/excel.go b/cve-vulner-manager/models/excel.go index ec4acb5f2945b51e1790f2aabb3d2d0b2f1ac48f..34e8f9c6459027c12299cec2da17b5ef145ccc02 100644 --- a/cve-vulner-manager/models/excel.go +++ b/cve-vulner-manager/models/excel.go @@ -10,6 +10,8 @@ import ( "cvevulner/common" ) +const splitLen = 2 + // ExcelExport the export excel row content model type ExcelExport struct { Num int64 @@ -44,8 +46,8 @@ func (e ExcelExport) ParseAnalysisVersion() map[string]string { split := strings.Split(e.AnalysisVersion, ",") for _, v := range split { item := strings.Split(strings.ReplaceAll(v, ":", ":"), ":") - if len(item) != 2 || item[1] == "" { - return nil + if len(item) != splitLen { + continue } result[item[0]] = item[1] @@ -67,38 +69,6 @@ func (e ExcelExport) GetReasonByVersion(v string) string { return "" } -func (e ExcelExport) WillFixVersion() []string { - var version []string - - for v, reason := range e.ParseAnalysisVersion() { - if reason == common.AnalysisWillFix { - version = append(version, v) - } - } - - return version -} - -func (e ExcelExport) IsWillFixVersion(v string) bool { - for _, version := range e.WillFixVersion() { - if version == v { - return true - } - } - - return false -} - -func (e ExcelExport) IsNotWillFixVersion(v string) bool { - for version, reason := range e.ParseAnalysisVersion() { - if v == version && reason != common.AnalysisWillFix { - return true - } - } - - return false -} - func (e ExcelExport) AffectType(v string) string { if !e.IsIssueWithAnalysisVersion() { return common.TypeUnaffected @@ -106,9 +76,15 @@ func (e ExcelExport) AffectType(v string) string { for version, reason := range e.ParseAnalysisVersion() { if v == version { + if reason == "" { + return common.TypeUnderInvestigation + } + if _, ok := common.AnalysisUnaffected[reason]; ok { return common.TypeUnaffected } + + break } } diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go index d023b42eecb1e8e5c12ce4b0464c5b0170638b7d..c9f0a1a98979a636946343ad25da7c223edd26b0 100644 --- a/cve-vulner-manager/models/issue.go +++ b/cve-vulner-manager/models/issue.go @@ -778,3 +778,14 @@ func QueryAuthTokenById(ati *AuthTokenInfo, colName ...string) error { err := o.Read(ati, colName...) return err } + +// IsIssueWithAnalysisVersion returns whether the issue has an analysis version. +func (t *IssueTemplate) IsIssueWithAnalysisVersion() bool { + return t.AnalysisVersion != "" +} + +// IsIssueComplete returns whether the issue is completed. +func (t *IssueTemplate) IsIssueComplete() bool { + const StatusCompleted = 3 + return t.Status == StatusCompleted +} diff --git a/cve-vulner-manager/task/cve.go b/cve-vulner-manager/task/cve.go index 6cc82376ca76ef57778a510226000a7aa3227069..cd6696056add230b59941368fd2d8d574e2b63e7 100644 --- a/cve-vulner-manager/task/cve.go +++ b/cve-vulner-manager/task/cve.go @@ -15,6 +15,7 @@ import ( "github.com/opensourceways/server-common-lib/utils" "cvevulner/common" + "cvevulner/cve-ddd/infrastructure/majunimpl" "cvevulner/cve-ddd/infrastructure/obsimpl" "cvevulner/taskhandler" @@ -99,14 +100,17 @@ func ReleaseUnaffectedCve() error { var unaffectcvrf = taskhandler.UnaffectCvrfSa{Xmlns: "http://www.icasi.org/CVRF/schema/cvrf/1.1", XmlnsCvrf: "http://www.icasi.org/CVRF/schema/cvrf/1.1"} cvrffileName := filepath.Join(dir, "cvrf-unaffected-cve-"+common.GetCurDate()+".xml") - du := beego.AppConfig.DefaultString("excel::v_pack_20_03_url", "") - csvPathList := strings.Split(du, ";") - for _, branch := range csvPathList { - branchs := strings.Split(branch, "@") - if len(branchs) > 0 && branchs[0] != "" { - taskhandler.UnaffectIssueProc(branchs[0], nil, nil, startTime, - accessToken, owner, &unaffectcvrf, unaffectYear, nil) - } + + majun := majunimpl.NewMajunImpl() + releasedBranch, err2 := majun.GetReleasedBranch() + if err2 != nil { + logs.Error("get released branch from majun failed:", err2.Error()) + return fmt.Errorf("get released branch from majun failed: %s", err2.Error()) + } + + for _, branch := range releasedBranch { + taskhandler.UnaffectIssueProc(branch, nil, nil, startTime, + accessToken, owner, &unaffectcvrf, unaffectYear, nil) } if len(unaffectcvrf.Vulnerability) == 0 { diff --git a/cve-vulner-manager/taskhandler/assist.go b/cve-vulner-manager/taskhandler/assist.go index b222e32ecf3cbe045716269a9cb87982e4cc992c..fa807f5bbadcb7de174130f3b27bfe717e3ff422 100644 --- a/cve-vulner-manager/taskhandler/assist.go +++ b/cve-vulner-manager/taskhandler/assist.go @@ -92,6 +92,10 @@ func GetAssignerOfOpeneuler(repo string) string { defer mutex.Unlock() today := time.Now().Format("20060102") + if len(assignerOfOpeneulerRepoCache) == 0 { + InitAssignerCache(today) + } + date, ok := assignerOfOpeneulerRepoCache[keyOfDate] if !ok || date != today { InitAssignerCache(today) @@ -104,8 +108,6 @@ func GetAssignerOfOpeneuler(repo string) string { ret = assigner } - logs.Error("get assigner of ", repo, " ,result is ", ret) - return ret } diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index e6bd3fea7a5f065e5c547e5fc26b783e2b499bea..05b3d718f230ae5987b8dddb5f9a90521e2ff617 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -16,7 +16,10 @@ import ( "sync" "time" + "k8s.io/apimachinery/pkg/util/sets" + "cvevulner/common" + "cvevulner/cve-ddd/infrastructure/majunimpl" "cvevulner/cve-timed-task/tabletask" "cvevulner/models" "cvevulner/util" @@ -2544,7 +2547,55 @@ func GetCveSecurityNotice(cveNumber, packageName string, flag bool) (bool, model return false, detail } -func GetCveProduct(cveNumber, packageName string, branch ...string) (bool, models.RespCveProduct) { +// IsAllProductReleased checks if all products associated with a given CVE number and package name are released. +// It returns true if all products are released, otherwise it returns false. +func IsAllProductReleased(cveNumber, packageName string) bool { + products := GetCveProduct(cveNumber, packageName) + + fixedProduct := sets.NewString() + for _, v := range products { + if v.IsFixed() { + fixedProduct.Insert(v.ProductName) + } + } + + majun := majunimpl.NewMajunImpl() + releasedBranch, err := majun.GetReleasedBranch() + if err != nil { + logs.Error("get released branch from majun failed: ", err.Error()) + return false + } + releasedBranchSets := sets.NewString(releasedBranch...) + + return releasedBranchSets.Equal(fixedProduct) +} + +// IsProductReleased checks if a product is released based on the CVE number, package name, and branch. +func IsProductReleased(cveNumber, packageName, branch string) (bool, string) { + products := GetCveProduct(cveNumber, packageName) + for _, v := range products { + if v.ProductName == branch { + return true, v.Status + } + } + + return false, "" +} + +// IsProductFixed checks if a product (branch) is fixed for a given CVE number and package name. +func IsProductFixed(cveNumber, packageName, branch string) bool { + products := GetCveProduct(cveNumber, packageName) + for _, v := range products { + if v.IsFixed() && v.ProductName == branch { + return true + } + } + + return false +} + +// GetCveProduct retrieves a list of CVE products based on the provided CVE number and package name. +func GetCveProduct(cveNumber, packageName string) []models.CveProduct { var detail models.RespCveProduct var urlS url.URL q := urlS.Query() @@ -2564,44 +2615,19 @@ func GetCveProduct(cveNumber, packageName string, branch ...string) (bool, model resp, err := http.Get(req.URL.String()) if err != nil { logs.Error("GetCveSecurityNotice, url: ", req.URL.String(), err) - return false, detail + return nil } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil || body == nil { - return false, detail + return nil } err = json.Unmarshal(body, &detail) if err != nil { - return false, detail - } - affectedBranchs := beego.AppConfig.String("cve::affected_branchs") - splitAffectedBranches := strings.Split(strings.ToLower(affectedBranchs), ",") - if len(detail.Result) >= len(splitAffectedBranches)-1 { - var p string - var f = true - for _, product := range detail.Result { - p += strings.ToLower(product.ProductName) + "," - } - for _, s := range splitAffectedBranches { - if !strings.Contains(p, s) { - f = false - break - } - } - if f { - return true, detail - } - } - if len(branch) > 0 { - for _, v := range detail.Result { - if strings.EqualFold(v.ProductName, branch[0]) { - return true, detail - } - } - return false, detail + return nil } - return len(detail.Result) >= len(splitAffectedBranches), detail + + return detail.Result } // FilterCveExported Filter exportable data @@ -2624,7 +2650,7 @@ func FilterCveExported() { center.IsExport = 1 models.UpdateVulnCenter(¢er, "is_export") dbLock.Unlock() - } else if productExist, _ := GetCveProduct(center.CveNum, center.PackName); productExist { + } else if b := IsAllProductReleased(center.CveNum, center.PackName); b { dbLock.Lock() center.IsExport = 1 models.UpdateVulnCenter(¢er, "is_export") diff --git a/cve-vulner-manager/taskhandler/excel.go b/cve-vulner-manager/taskhandler/excel.go index 1c2d39976b8ff24b0f426a44b25b87dad2e69e28..e768c2271f59a13d40a359bf419dcd0d11877fe0 100644 --- a/cve-vulner-manager/taskhandler/excel.go +++ b/cve-vulner-manager/taskhandler/excel.go @@ -632,8 +632,9 @@ func affectBrachRep(xmlp *models.ExcelExport, affectBranch string) bool { func FindUnaffectBrach(xmlp *models.ExcelExport, affectBranch, accessToken, owner string) bool { branchArry, _ := GetBranchesInfo(accessToken, owner, xmlp.OwnedComponent, 1) + // 包含原因分析字段的新issue,所有分支都要发布 if xmlp.IsIssueWithAnalysisVersion() { - return xmlp.IsNotWillFixVersion(affectBranch) + return true } affectBool := false @@ -1182,19 +1183,18 @@ func UnaffectIssueProc(affectBranch string, cvrfFileList map[string][]string, data, err := getDataUnaffect(startTime, cves) if len(data) > 0 { for _, v := range data { + // 对于旧数据,仍然按照未完成就不发布的逻辑 + if !v.IsIssueWithAnalysisVersion() && !v.IsIssueComplete() { + continue + } + + var status string + var released bool issueExist, _ := GetCveSecurityNotice(v.CveNum, v.Repo, true) if issueExist { - if productExist, _ := GetCveProduct(v.CveNum, v.Repo); productExist { - var center models.VulnCenter - center.CveId = v.CveId - centErr := models.GetVulnCenterByCid(¢er, "cve_id") - if centErr == nil { - center.IsExport = 1 - models.UpdateVulnCenter(¢er, "is_export") - } - continue - } - if ok, _ := GetCveProduct(v.CveNum, v.Repo, affectBranch); ok { + // 没发布过的分支都要发布 + released, status = IsProductReleased(v.CveNum, v.Repo, affectBranch) + if released && status != common.TypeUnderInvestigation { continue } } @@ -1211,6 +1211,14 @@ func UnaffectIssueProc(affectBranch string, cvrfFileList map[string][]string, if filterFixBranch(&vx, vx.CveNum, affectBranch) { continue } + + // 发布过的分支状态如果是调查中,要被其他状态覆盖;覆盖时,如果该分支还是调查中,则忽略,不生成 + if vx.IsIssueWithAnalysisVersion() && + status == common.TypeUnderInvestigation && + vx.GetReasonByVersion(affectBranch) == "" { + continue + } + affectBool := FindUnaffectBrach(&vx, affectBranch, accessToken, owner) if affectBool { logs.Info("Unaffected version, data: ", vx.CveNum, vx.OwnedComponent, vx.AffectProduct) @@ -1250,7 +1258,8 @@ func filterDataInSlice(data string, filterList []string) bool { // if cve exist affected and label exist CVE/FIXED return true func filterFixBranch(data *models.ExcelExport, cve, branch string) (has bool) { if data.IsIssueWithAnalysisVersion() { - return data.IsWillFixVersion(branch) + // 包含原因分析字段的新issue,所有分支都要发布 + return false } has = false @@ -1299,17 +1308,7 @@ func affectIssueProc(v IssueAndPkg, affectBranch string, cvexml *[]CveXml, // Check whether the cve data has been released sa issueExist, _ := GetCveSecurityNotice(tpl.CveNum, tpl.Repo, true) if issueExist { - if productExist, _ := GetCveProduct(tpl.CveNum, tpl.Repo); productExist { - var center models.VulnCenter - center.CveId = tpl.CveId - centErr := models.GetVulnCenterByCid(¢er, "cve_id") - if centErr == nil { - center.IsExport = 1 - models.UpdateVulnCenter(¢er, "is_export") - } - continue - } - if ok, _ := GetCveProduct(tpl.CveNum, v.Repo, affectBranch); ok { + if IsProductFixed(tpl.CveNum, v.Repo, affectBranch) { continue } }