From 4c46d4d46525142c4916fd70cc0230e3e2e71331 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Sat, 12 Oct 2024 09:17:56 +0800 Subject: [PATCH 1/4] some optimize --- cve-vulner-manager/conf/product_app.conf | 2 +- cve-vulner-manager/cve-ddd/app/bulletin.go | 9 +++++++++ cve-vulner-manager/cve-ddd/app/coldpatch.go | 13 +++---------- .../cve-ddd/infrastructure/backendimpl/impl.go | 12 +++++++++++- .../cve-ddd/infrastructure/testresultimpl/impl.go | 7 ++++++- 5 files changed, 30 insertions(+), 13 deletions(-) diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 080c8fb..04e1dc5 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -132,7 +132,7 @@ syncissuedateflag = 1 syncissuedate = 0 0 7,13 * * * releaseUnaffectedCveflag = 1 -releaseUnaffectedCve = 0 0 11 * * * +releaseUnaffectedCve = 0 0 11 * * 5 [gitee] diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index 569224f..2444441 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -123,6 +123,8 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin uploadDir := b.generateUploadDir() + publishDir := beego.AppConfig.String("obs::download_cvrf_dir") + bulletins := testedCves.GenerateBulletins() var updateFixedFiles []string @@ -146,6 +148,13 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin continue } + // 这步操作本来是维护人员从uploadDir下载下来，人工确认无误后，再上传到publishDir目录 + // 后来错误率降低后直接省略该步骤，但还是会人工核验 + publishPath := publishDir + v.PathAppendToIndexFile() + if err1 = b.obs.Upload(publishPath, xmlData); err1 != nil { + b.log.Errorf("upload cold patch %s to cvrf dir failed: %v", v.Identification, err1) + } + updateFixedFiles = append(updateFixedFiles, v.PathAppendToIndexFile()) b.uploadUpdateInfoFile(&v) diff --git a/cve-vulner-manager/cve-ddd/app/coldpatch.go b/cve-vulner-manager/cve-ddd/app/coldpatch.go index d6a30d5..2f246fa 100644 --- a/cve-vulner-manager/cve-ddd/app/coldpatch.go +++ b/cve-vulner-manager/cve-ddd/app/coldpatch.go @@ -271,26 +271,19 @@ func (c *coldPatchService) filterData( continue } - // 3.pr合入时间超过半年的不处理（为了减少数据量） - sixMonthAgo := time.Now().AddDate(0, -6, 0) - if !mergeAt.After(sixMonthAgo) { - c.log.Errorf("pr merge time of %s %s check failed", branch, data.ToLogString()) - continue - } - buildTime, err1 := c.rpm.GetBuildTime(branch, data.Issue.Repo) if err1 != nil { c.log.Errorf("get build time of %s %s error:%v", branch, data.ToLogString(), err1) continue } - // 4.pr合入时间必须在工程构建时间之后（pr合入后，工程会重新构建，并上传至latest_rpms仓库，如果工程因其他原因未构建，就不需要转测了） + // 3.pr合入时间必须在工程构建时间之后（pr合入后，工程会重新构建，并上传至latest_rpms仓库，如果工程因其他原因未构建，就不需要转测了） if !buildTime.After(mergeAt) { c.log.Errorf("build time check failed of %s %s", branch, data.ToLogString()) continue } - // 5.在版本分支发布日期之前合入的pr不处理，因为已经随着新版本一起发布并修复了 + // 4.在版本分支发布日期之前合入的pr不处理，因为已经随着新版本一起发布并修复了 releaseTimeOfBranch, ok := c.releaseDate.Load(branch) if ok { releaseTime := releaseTimeOfBranch.(time.Time) @@ -310,7 +303,7 @@ func (c *coldPatchService) filterData( return false, err } - // 6.已经发布到官网的分支不用再处理了 + // 5.已经发布到官网的分支不用再处理了 publishSets := sets.New(publishedBranch...) diff := needToHandleBranch.Difference(publishSets) if len(diff) == 0 { diff --git a/cve-vulner-manager/cve-ddd/infrastructure/backendimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/backendimpl/impl.go index fad14bf..97c9d27 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/backendimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/backendimpl/impl.go @@ -9,6 +9,10 @@ import ( "github.com/opensourceways/server-common-lib/utils" ) +const ( + statusFixed = "Fixed" +) + func NewBackendImpl() backendImpl { return backendImpl{ client: utils.NewHttpClient(3), @@ -30,6 +34,10 @@ type Result struct { Status string `json:"status"` } +func (r Result) IsFixed() bool { + return r.Status == statusFixed +} + func (impl backendImpl) PublishedInfo(cveNum, packageName string) ([]string, error) { url := fmt.Sprintf("%s/api-cve/cve-security-notice-server/cvedatabase/getCVEProductPackageList?cveId=%s&packageName=%s", beego.AppConfig.String("reflink::openeuler_web"), cveNum, packageName, @@ -52,7 +60,9 @@ func (impl backendImpl) PublishedInfo(cveNum, packageName string) ([]string, err var ret []string for _, item := range v.Result { - ret = append(ret, item.ProductName) + if item.IsFixed() { + ret = append(ret, item.ProductName) + } } return ret, nil diff --git a/cve-vulner-manager/cve-ddd/infrastructure/testresultimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/testresultimpl/impl.go index e40df85..624495d 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/testresultimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/testresultimpl/impl.go @@ -179,7 +179,12 @@ func (impl *testResultImpl) parseContent(content []byte, fromEpol bool) map[stri t = append(t, rpm{Name: v, IsEpol: fromEpol}) } - componentAndRpm[line[0]] = t + splitComponent := strings.Split(line[0], ":") + if len(splitComponent) == 0 { + continue + } + + componentAndRpm[splitComponent[0]] = append(componentAndRpm[splitComponent[0]], t...) } return componentAndRpm -- Gitee From b23ca94e2e72d3542b78ad50bd84914b9329c804 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Sat, 12 Oct 2024 16:09:08 +0800 Subject: [PATCH 2/4] optimize score --- cve-vulner-manager/cve-ddd/app/bulletin.go | 9 --- cve-vulner-manager/taskhandler/common.go | 10 +--- cve-vulner-manager/taskhandler/cve.go | 64 ++++++++++++---------- 3 files changed, 39 insertions(+), 44 deletions(-) diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index 2444441..569224f 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -123,8 +123,6 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin uploadDir := b.generateUploadDir() - publishDir := beego.AppConfig.String("obs::download_cvrf_dir") - bulletins := testedCves.GenerateBulletins() var updateFixedFiles []string @@ -148,13 +146,6 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin continue } - // 这步操作本来是维护人员从uploadDir下载下来，人工确认无误后，再上传到publishDir目录 - // 后来错误率降低后直接省略该步骤，但还是会人工核验 - publishPath := publishDir + v.PathAppendToIndexFile() - if err1 = b.obs.Upload(publishPath, xmlData); err1 != nil { - b.log.Errorf("upload cold patch %s to cvrf dir failed: %v", v.Identification, err1) - } - updateFixedFiles = append(updateFixedFiles, v.PathAppendToIndexFile()) b.uploadUpdateInfoFile(&v) diff --git a/cve-vulner-manager/taskhandler/common.go b/cve-vulner-manager/taskhandler/common.go index 196b728..7cf3429 100644 --- a/cve-vulner-manager/taskhandler/common.go +++ b/cve-vulner-manager/taskhandler/common.go @@ -869,16 +869,12 @@ func CreateIssueBody(accessToken, owner, path, assignee string, its models.IssueTemplate, flag int, issueType, pkgLink string, brandArray []string) string { var issueOption IssueOptions - scoreType := "" + scoreType := "3.0" if sc.ScoreType == "v2" { scoreType = "2.0" - } else { - scoreType = "3.0" - } - nvdType := "2.0" - if score != "0.0" { - nvdType = "3.0" } + nvdType := scoreType + if len(brandArray) == 0 { switch cve.OrganizationID { case util.Openeuler, util.OpenGauss: diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index 8f44985..e6bd3fe 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -778,6 +778,30 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef, repoNme string, return true, nil } +func getScoreV2(impactId int64) models.OriginUpstreamImpactScoreV2 { + cveScore, ok := models.QueryCveScore(impactId, "v2") + if ok { + scoreV2, ok2 := models.QueryCveCvssV2(cveScore.ScoreId) + if ok2 { + return scoreV2 + } + } + + return models.OriginUpstreamImpactScoreV2{} +} + +func getScoreV3(impactId int64) models.OriginUpstreamImpactScoreV3 { + cveScore, ok := models.QueryCveScore(impactId, "v3") + if ok { + scoreV3, ok2 := models.QueryCveCvssV3(cveScore.ScoreId) + if ok2 { + return scoreV3 + } + } + + return models.OriginUpstreamImpactScoreV3{} +} + // Synchronize the data returned by the Chinese Academy of Sciences func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) (bool, error) { if cveData.Ids == "" || cveData.CveNum == "" { @@ -831,37 +855,21 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 1", 3) return false, errors.New("数据错误，暂时不处理") } - scopeType := "v3" - //var cveScV2 models.OriginUpstreamImpactScoreV2 - cveScore, ok := models.QueryCveScore(cveImpact.ImpactId, "v3") - if !ok { - scopeType = "v2" - } - cveScV3, ok := models.QueryCveCvssV3(cveScore.ScoreId) - if !ok { - scopeType = "v2" + + cveScV2 := getScoreV2(cveImpact.ImpactId) + cveScV3 := getScoreV3(cveImpact.ImpactId) + if cveScV3.V3Id == 0 && cveScV2.V2Id == 0 { + logs.Error("GenCveVuler, can not get score of data: ", cveData) + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 3", 3) + return false, errors.New("数据错误，暂时不处理") } - if cveScV3.VectorString == "" || len(cveScV3.VectorString) == 0 || cveScV3.BaseScore == 0.0 { + + scopeType := "v3" + if cveScV3.BaseScore == 0 && cveScV2.BaseScore > 0 { scopeType = "v2" } - cveScoreV2, ok2 := models.QueryCveScore(cveImpact.ImpactId, "v2") - if !ok2 { - if scopeType == "v2" { - logs.Error("GenCveVuler, QueryCveScore3, data: ", cveData, ",cveImpact: ", cveImpact) - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) - UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 2", 3) - return false, errors.New("数据错误，暂时不处理") - } - } - cveScV2, okV2 := models.QueryCveCvssV2(cveScoreV2.ScoreId) - if !okV2 { - if scopeType == "v2" { - logs.Error("GenCveVuler, QueryCveCvssV2, data: ", cveData, ",cveScore: ", cveScore) - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) - UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 3", 3) - return false, errors.New("数据错误，暂时不处理") - } - } + packNameMap := map[string]string{} packNameList := []string{} if cveData.PackName != "" && len(cveData.PackName) > 0 { -- Gitee From 0f7e92c00884325ff7c014c1d06b62e03772e9e1 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Mon, 14 Oct 2024 10:53:18 +0800 Subject: [PATCH 3/4] stop issue to template --- cve-vulner-manager/conf/product_app.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 04e1dc5..8609bc6 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -80,7 +80,7 @@ cveflag = 1 getcve = 0 */40 * * * * oricveflag = 1 oricvecheck = 0 0 2 * * * -getissueflag = 1 +getissueflag = 2 getissue = 0 20 1 * * * issueflag = 1 createissue = 0 0 6 * * * -- Gitee From 0f953b46ee8936749f694ccbb0c341880d539b4e Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Mon, 14 Oct 2024 15:06:58 +0800 Subject: [PATCH 4/4] fix bug of create time --- cve-vulner-manager/taskhandler/common.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/cve-vulner-manager/taskhandler/common.go b/cve-vulner-manager/taskhandler/common.go index 7cf3429..db2bc95 100644 --- a/cve-vulner-manager/taskhandler/common.go +++ b/cve-vulner-manager/taskhandler/common.go @@ -7,6 +7,7 @@ import ( "os" "strconv" "strings" + "time" "cvevulner/common" "cvevulner/models" @@ -918,17 +919,13 @@ func CreateIssueBody(accessToken, owner, path, assignee string, affectedVersion := AffectVersionExtract(brandArray, its.AffectedVersion, cve.PackName, cve.OrganizationID) abiVersion := AffectVersionExtract(brandArray, its.AbiVersion, cve.PackName, cve.OrganizationID) updateTemplateAbi(brandArray, its) - logs.Info("its.CreateTime: ", its.CreateTime) - updateTime := its.CreateTime.Format(common.DATE_FORMAT) - if len(updateTime) >= 19 && its.TemplateId > 0 { - updateTime = common.TimeConverStr(updateTime[:19]) - } else { - updateTime = common.GetLocalCurTime() - } - logs.Info("cve.UpdateTime.String(): ", updateTime, its.CreateTime, cve.CreateTime) - if updateTime != "" && len(updateTime) > 19 { - updateTime = updateTime[:19] + + createdTime := time.Now() + if its.TemplateId > 0 { + createdTime = its.CreateTime } + updateTime := createdTime.Format(common.DATE_FORMAT) + StatusName := "" if its.StatusName != "" && len(its.StatusName) > 1 { if its.StatusName == "待办的" || its.StatusName == "开启的" || -- Gitee