From 18cd91f532b220f9e2affacfafc304341f4a4234 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Fri, 18 Jul 2025 15:00:34 +0800 Subject: [PATCH 1/3] optimize generate bulletin --- cve-vulner-manager/cve-ddd/app/bulletin.go | 34 +++++++++++++--------- cve-vulner-manager/cve-ddd/domain/cve.go | 10 +++++-- 2 files changed, 29 insertions(+), 15 deletions(-) diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index 15e6a09..131679a 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -108,11 +108,11 @@ func (b *bulletinService) GenerateBulletins(cmd CmdToGenerate) (string, error) { b.log.Errorf("num of cveNum %d and num of cves %d does not match", len(cveNum), len(cves)) } - // 用请求参数的分支信息过滤，只处理cve指定的分支 - cves = b.filterByRequestParam(cves, cmd) + // 用请求参数分支和在维分支信息过滤，只处理cve指定的分支 + cves = b.filterByRequestParam(cves, cmd, handleBranch) // 用关联pr并且合入的分支进行过滤 - cves = b.filterByRelatedPR(cves, handleBranch) + cves = b.filterByRelatedPR(cves) // 用已发布的分支进行过滤 cves = b.filterByPublishedBranch(cves) @@ -201,22 +201,27 @@ func (b *bulletinService) getIndexContent() (string, error) { return string(content), err } -func (b *bulletinService) filterByRequestParam(cves domain.Cves, cmd CmdToGenerate) domain.Cves { +func (b *bulletinService) filterByRequestParam(cves domain.Cves, cmd CmdToGenerate, handleBranch []string) domain.Cves { var filteredCves domain.Cves for _, v := range cves { if branches, ok := cmd.CveNum[v.CveNum]; ok { - v.AffectedVersion = branches - filteredCves = append(filteredCves, v) + v.SetAffectVersionWithIntersection(handleBranch) + v.SetAffectVersionWithIntersection(branches) + b.log.Infof("filter by request param, %s: %v", v.CveNum, v.AffectedVersion) + + if len(v.AffectedVersion) == 0 { + continue + } + + filteredCves = append(filteredCves, v) } } return filteredCves } -func (b *bulletinService) filterByRelatedPR(cves domain.Cves, handleBranch []string) domain.Cves { - handleBranchSet := sets.NewString(handleBranch...) - +func (b *bulletinService) filterByRelatedPR(cves domain.Cves) domain.Cves { var filteredCves domain.Cves for _, v := range cves { prs, _, err := b.getRelatedPR(v.ColdIssue) @@ -225,7 +230,7 @@ func (b *bulletinService) filterByRelatedPR(cves domain.Cves, handleBranch []str continue } - relatedPrSets := sets.NewString() + var relatedPRBranches []string for _, pr := range prs { if pr.Base.Repo.Namespace.Path != defaultOwner { continue @@ -256,16 +261,19 @@ func (b *bulletinService) filterByRelatedPR(cves domain.Cves, handleBranch []str } } - relatedPrSets.Insert(branch) + relatedPRBranches = append(relatedPRBranches, branch) } - intersection := handleBranchSet.Intersection(relatedPrSets) - v.AffectedVersion = intersection.UnsortedList() + v.SetAffectVersionWithIntersection(relatedPRBranches) b.log.Infof("the affected version of [%s %s] after pr filter are %v", v.CveNum, v.ColdIssue.Number, v.AffectedVersion, ) + if len(v.AffectedVersion) == 0 { + continue + } + filteredCves = append(filteredCves, v) } diff --git a/cve-vulner-manager/cve-ddd/domain/cve.go b/cve-vulner-manager/cve-ddd/domain/cve.go index e8163b8..645ad5a 100644 --- a/cve-vulner-manager/cve-ddd/domain/cve.go +++ b/cve-vulner-manager/cve-ddd/domain/cve.go @@ -37,8 +37,8 @@ type Issue struct { Repo string // in src-openeuler, repo == package, name == component } -func (d Cve) isAffectVersion(version string) bool { - for _, v := range d.AffectedVersion { +func (c *Cve) isAffectVersion(version string) bool { + for _, v := range c.AffectedVersion { if v == version { return true } @@ -47,6 +47,12 @@ func (d Cve) isAffectVersion(version string) bool { return false } +func (c *Cve) SetAffectVersionWithIntersection(branches []string) { + s1 := sets.NewString(c.AffectedVersion...) + s2 := sets.NewString(branches...) + c.AffectedVersion = s1.Intersection(s2).List() +} + // FilterAffectVersion 只处理需要发布公告的版本分支 func (cs Cves) FilterAffectVersion() { for k, v := range cs { -- Gitee From 93c4e5fe5931171d64012d808efe3085739c4868 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Mon, 21 Jul 2025 14:58:38 +0800 Subject: [PATCH 2/3] fix release cve bug --- cve-vulner-manager/taskhandler/excel.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cve-vulner-manager/taskhandler/excel.go b/cve-vulner-manager/taskhandler/excel.go index ba84315..0cc675c 100644 --- a/cve-vulner-manager/taskhandler/excel.go +++ b/cve-vulner-manager/taskhandler/excel.go @@ -514,8 +514,7 @@ func UnaffectIssueProc(affectBranch string, cvrfFileList map[string][]string, } // 发布过的除fixed可能被修改，官网数据需要被覆盖，状态一致则忽略 - if vx.IsIssueWithAnalysisVersion() && - vx.AffectType(affectBranch) == status { + if vx.AffectType(affectBranch) == status { continue } -- Gitee From 3a23edd983f6ee1543e2d1d5d3a3f271b50ea0e7 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Mon, 4 Aug 2025 11:17:55 +0800 Subject: [PATCH 3/3] optimize create issue --- cve-vulner-manager/models/issue.go | 4 +- cve-vulner-manager/task/issuetask.go | 264 +++++++++++++-------------- 2 files changed, 134 insertions(+), 134 deletions(-) diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go index 692d99e..1c3e036 100644 --- a/cve-vulner-manager/models/issue.go +++ b/cve-vulner-manager/models/issue.go @@ -93,10 +93,10 @@ func QueryIssue(days string, prcnum int, list []string) ([]VulnCenter, error) { s = s[:len(s)-1] } num, err = o.Raw(fmt.Sprintf("select * from cve_vuln_center where cve_num in (%s) and cve_status in (0, 1) "+ - "order by cve_id asc limit %v", s, prcnum)).QueryRows(&vc) + "order by cve_id asc", s)).QueryRows(&vc) } else { num, err = o.Raw("select * from cve_vuln_center where cve_status in (0, 1) or (cve_status = 3 and create_time > ?) "+ - "order by cve_id asc limit ?", "2025-07-01", prcnum).QueryRows(&vc) + "order by cve_id asc ", "2025-07-01").QueryRows(&vc) } if err == nil && num > 0 { logs.Info("QueryIssue, cve_vuln_center, search result: ", vc) diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go index 96c6e17..f23e925 100644 --- a/cve-vulner-manager/task/issuetask.go +++ b/cve-vulner-manager/task/issuetask.go @@ -154,154 +154,154 @@ compareRes: } func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, flag int, list []string) error { - for { - cveData, err := models.QueryIssue(beforeTime, prcnum, list) - if err == nil && len(cveData) > 0 { - logs.Info("cveData: ", cveData) - //ss := rand.Int31n(10) - //time.Sleep(time.Second * time.Duration(ss)) - } else { - logs.Info("addUnlimitedIssue, No cve data can be used, current time: ", common.GetCurTime(), ", err: ", err) - return err + cveData, err := models.QueryIssue(beforeTime, prcnum, list) + if err == nil && len(cveData) > 0 { + logs.Info("cveData: ", cveData) + //ss := rand.Int31n(10) + //time.Sleep(time.Second * time.Duration(ss)) + } else { + logs.Info("addUnlimitedIssue, No cve data can be used, current time: ", common.GetCurTime(), ", err: ", err) + return err + } + for index, issueValue := range cveData { + if models.FilterOldData(issueValue.CveNum) { + var it models.IssueTemplate + it.CveId = issueValue.CveId + it.CveNum = issueValue.CveNum + templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") + if templateErr != nil { + models.UpdateIssueStatus(issueValue, 12) + continue + } } - for index, issueValue := range cveData { - if models.FilterOldData(issueValue.CveNum) { - var it models.IssueTemplate - it.CveId = issueValue.CveId - it.CveNum = issueValue.CveNum - templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") - if templateErr != nil { - models.UpdateIssueStatus(issueValue, 12) - continue - } + logs.Info("addUnlimitedIssue, Currently processing：", index, ",cve data, cveNum: ", issueValue.CveNum) + // add mutex + lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) + if !lockErr { + logs.Info("addUnlimitedIssue, The current cve is processing, continue to process the next data, "+ + "err: ", lockErr, ",data: ", issueValue) + continue + } + owner, accessToken := common.GetOwnerAndToken(issueValue.CveNum, issueValue.OrganizationID) + // Determine whether the issue has been processed + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName, issueValue.OrganizationID) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝" || goi.IssueState == "已挂起" { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("addUnlimitedIssue, The cve data has already been submitted to the issue, "+ + "no need to submit repeatedly, cveData: ", issueValue) + ErrorCveStatistics("CVE已创建过issue, 且已归档", issueValue, 1) + continue } - logs.Info("addUnlimitedIssue, Currently processing：", index, ",cve data, cveNum: ", issueValue.CveNum) - // add mutex - lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) - if !lockErr { - logs.Info("addUnlimitedIssue, The current cve is processing, continue to process the next data, "+ - "err: ", lockErr, ",data: ", issueValue) + } + if issueValue.OrganizationID == 1 { + se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} + seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") + if seError == nil && se.Id > 0 { + models.UpdateIssueStatus(issueValue, 5) + logs.Info("addUnlimitedIssue, The current issue does not need to be processed, "+ + "it has been processed, cveData: ", issueValue) + ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) continue } - owner, accessToken := common.GetOwnerAndToken(issueValue.CveNum, issueValue.OrganizationID) - // Determine whether the issue has been processed - goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName, issueValue.OrganizationID) - if oks { - if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || - goi.State == "已完成" || goi.State == "已拒绝" || goi.IssueState == "已挂起" { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("addUnlimitedIssue, The cve data has already been submitted to the issue, "+ - "no need to submit repeatedly, cveData: ", issueValue) - ErrorCveStatistics("CVE已创建过issue, 且已归档", issueValue, 1) - continue - } + // Determine whether cve has been processed + exist, detail := taskhandler.GetCveSecurityNotice(issueValue.CveNum, issueValue.PackName, true) + if exist && detail.Result.IsFixed() { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("addUnlimitedIssue, The cve data has been displayed on the official website, "+ + "no need to submit an issue on git, cveData: ", issueValue) + ErrorCveStatistics("CVE已经在官网展示, 已修复", issueValue, 1) + continue } - if issueValue.OrganizationID == 1 { - se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} - seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") - if seError == nil && se.Id > 0 { - models.UpdateIssueStatus(issueValue, 5) - logs.Info("addUnlimitedIssue, The current issue does not need to be processed, "+ - "it has been processed, cveData: ", issueValue) - ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) - continue - } - // Determine whether cve has been processed - exist, detail := taskhandler.GetCveSecurityNotice(issueValue.CveNum, issueValue.PackName, true) - if exist && detail.Result.IsFixed() { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("addUnlimitedIssue, The cve data has been displayed on the official website, "+ - "no need to submit an issue on git, cveData: ", issueValue) - ErrorCveStatistics("CVE已经在官网展示, 已修复", issueValue, 1) - continue - } - if issueValue.Status == 0 && flag != 2 && len(issueValue.CveVersion) > 0 { - branchBool := CheckCveIssueBranch(issueValue.PackName, issueValue.CveVersion) - if !branchBool { - models.UpdateIssueStatus(issueValue, 8) - logs.Info("addUnlimitedIssue, "+ - "The current repo branch and version information do not match, "+ - "cveData: ", issueValue) - ErrorCveStatistics("仓库分支和版本信息不想符合", issueValue, 2) - continue - } - } - } else if issueValue.OrganizationID == 2 { - ogc := models.OpenGaussCveList{CveNum: issueValue.CveNum, PackName: issueValue.RepoName, Status: 3} - ogcErr := models.QueryReleaseCve(&ogc, "CveNum", "PackName", "Status") - if ogc.Id > 0 { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("addUnlimitedIssue, The cve data has been displayed on the official website, "+ - "no need to submit an issue on git, cveData: ", issueValue, ogcErr) - ErrorCveStatistics("CVE已经在官网展示, 已修复", issueValue, 1) + if issueValue.Status == 0 && flag != 2 && len(issueValue.CveVersion) > 0 { + branchBool := CheckCveIssueBranch(issueValue.PackName, issueValue.CveVersion) + if !branchBool { + models.UpdateIssueStatus(issueValue, 8) + logs.Info("addUnlimitedIssue, "+ + "The current repo branch and version information do not match, "+ + "cveData: ", issueValue) + ErrorCveStatistics("仓库分支和版本信息不想符合", issueValue, 2) continue } } - var it models.IssueTemplate - it.CveId = issueValue.CveId - it.CveNum = issueValue.CveNum - templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") - if templateErr != nil { - logs.Warn("addUnlimitedIssue, templateErr:", templateErr, ", CveNum: ", issueValue.CveNum) + } else if issueValue.OrganizationID == 2 { + ogc := models.OpenGaussCveList{CveNum: issueValue.CveNum, PackName: issueValue.RepoName, Status: 3} + ogcErr := models.QueryReleaseCve(&ogc, "CveNum", "PackName", "Status") + if ogc.Id > 0 { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("addUnlimitedIssue, The cve data has been displayed on the official website, "+ + "no need to submit an issue on git, cveData: ", issueValue, ogcErr) + ErrorCveStatistics("CVE已经在官网展示, 已修复", issueValue, 1) + continue } - // Import cve as data after 2018 - cveNumList := strings.Split(issueValue.CveNum, "-") - if cveNumList != nil && len(cveNumList) > 1 { - cveYears, yearErr := strconv.Atoi(cveNumList[1]) - if yearErr == nil { - if issueValue.DataSource == 1 || issueValue.DataSource == 5 { - years = toolYears - } else if issueValue.DataSource == 3 { - years = manYears - } - if cveYears <= years { - yearFlag := false - if it.TemplateId > 0 && len(it.IssueNum) > 0 { - issueErr, issueBody := taskhandler.GetGiteeIssue(accessToken, owner, issueValue.PackName, it.IssueNum) - if issueErr == nil && len(issueBody) > 0 { - yearFlag = true - } - } - if !yearFlag { - models.UpdateIssueStatus(issueValue, 4) - logs.Info("addUnlimitedIssue, cve: ", issueValue.CveNum, ",Need to be greater than： ", - years, ",Otherwise, there is no need to submit an issue on git, cveData: ", issueValue) - ErrorCveStatistics("CVE年限受限", issueValue, 2) - continue + } + var it models.IssueTemplate + it.CveId = issueValue.CveId + it.CveNum = issueValue.CveNum + templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") + if templateErr != nil { + logs.Warn("addUnlimitedIssue, templateErr:", templateErr, ", CveNum: ", issueValue.CveNum) + } + // Import cve as data after 2018 + cveNumList := strings.Split(issueValue.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if issueValue.DataSource == 1 || issueValue.DataSource == 5 { + years = toolYears + } else if issueValue.DataSource == 3 { + years = manYears + } + if cveYears <= years { + yearFlag := false + if it.TemplateId > 0 && len(it.IssueNum) > 0 { + issueErr, issueBody := taskhandler.GetGiteeIssue(accessToken, owner, issueValue.PackName, it.IssueNum) + if issueErr == nil && len(issueBody) > 0 { + yearFlag = true } } + if !yearFlag { + models.UpdateIssueStatus(issueValue, 4) + logs.Info("addUnlimitedIssue, cve: ", issueValue.CveNum, ",Need to be greater than： ", + years, ",Otherwise, there is no need to submit an issue on git, cveData: ", issueValue) + ErrorCveStatistics("CVE年限受限", issueValue, 2) + continue + } } } - // Process each piece of cve data - if len(it.IssueNum) == 0 { - issueValue.Status = 2 - mutex.Lock() - err := ProcIssue(issueValue, accessToken, owner) - mutex.Unlock() - if err != nil { - logs.Error("addUnlimitedIssue, Failed to create issue, cvenum: ", - issueValue.CveNum, "err,err: ", err) - errDesc := fmt.Sprintf("%v", err) - ErrorCveStatistics(errDesc, issueValue, 2) - continue - } - } else { - issueValue.Status = 2 - mutex.Lock() - err := ProcUpdateIssue(issueValue, accessToken, owner) - mutex.Unlock() - if err != nil { - logs.Error("addUnlimitedIssue, Failed to update issue, cvenum: ", - issueValue.CveNum, "err,err: ", err) - errDesc := fmt.Sprintf("%v", err) - ErrorCveStatistics(errDesc, issueValue, 2) - continue - } + } + // Process each piece of cve data + if len(it.IssueNum) == 0 { + issueValue.Status = 2 + mutex.Lock() + err := ProcIssue(issueValue, accessToken, owner) + mutex.Unlock() + if err != nil { + logs.Error("addUnlimitedIssue, Failed to create issue, cvenum: ", + issueValue.CveNum, "err,err: ", err) + errDesc := fmt.Sprintf("%v", err) + ErrorCveStatistics(errDesc, issueValue, 2) + continue + } + } else { + issueValue.Status = 2 + mutex.Lock() + err := ProcUpdateIssue(issueValue, accessToken, owner) + mutex.Unlock() + if err != nil { + logs.Error("addUnlimitedIssue, Failed to update issue, cvenum: ", + issueValue.CveNum, "err,err: ", err) + errDesc := fmt.Sprintf("%v", err) + ErrorCveStatistics(errDesc, issueValue, 2) + continue } - // Collect issue record data - ErrorCveStatistics("success", issueValue, 1) } + // Collect issue record data + ErrorCveStatistics("success", issueValue, 1) } + + return nil } func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears int) error { -- Gitee