From b79403a16469700fac92d28d0cbf615fb225bd63 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Thu, 12 Dec 2024 19:15:13 +0800 Subject: [PATCH 1/9] fix bug of trim --- .../cve-ddd/infrastructure/updateinfoimpl/repodata.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/repodata.go b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/repodata.go index cac9c28..60b6271 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/repodata.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/repodata.go @@ -52,7 +52,7 @@ func (impl updateInfoImpl) getEpochOfRpm(branch, date, arch string) (map[string] epochOfRpm := make(map[string]string) for _, v := range primaryXml.Package { - rpmName := strings.Trim(v.Location.Href, "Packages/") + rpmName := strings.TrimPrefix(v.Location.Href, "Packages/") epochOfRpm[rpmName] = v.Version.Epoch } -- Gitee From c30b754a4406a09939a178cdb7f32269dcc4445a Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Fri, 13 Dec 2024 15:16:08 +0800 Subject: [PATCH 2/9] optimize updateinfo dir --- cve-vulner-manager/cve-ddd/app/bulletin.go | 5 +++-- cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go | 6 ++++-- .../cve-ddd/domain/service/service.go | 13 +++++++------ 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index 0c5b514..7b9dab9 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -31,7 +31,8 @@ const ( fileIndex = "index.txt" fileUpdateFixed = "update_fixed.txt" - updateinfoRootDir = "earlyupdateinfo/" + updateInfoEarlyDir = "earlyupdateinfo/" + updateInfoRootDir = "updateinfo/" ) type BulletinService interface { @@ -151,7 +152,7 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin updateFixedFiles = append(updateFixedFiles, v.PathAppendToIndexFile()) - service.UploadUpdateInfoFile(b.obs, b.updateinfo, b.log, &v, date, updateinfoRootDir) + service.UploadUpdateInfoFile(b.obs, b.updateinfo, b.log, &v, date, updateInfoRootDir, updateInfoEarlyDir) } b.uploadIndexAndFixed(uploadDir, indexContent, updateFixedFiles) diff --git a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go index fe52ee1..a196c1c 100644 --- a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go @@ -20,7 +20,8 @@ import ( const ( fileHotPatch = "update_hot_patch.txt" - hotPatchUpdateInfoDir = "hotpatch-early-updateinfo/" + hotPatchUpdateInfoDir = "updateinfo-hotpatch/" + hotPatchEarlyUpdateInfoDir = "hotpatch-early-updateinfo/" ) type RefactorHotPatchService interface { @@ -130,7 +131,8 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir, date string) erro cvesForUpdateInfo = append(cvesForUpdateInfo, b.Cves...) - service.UploadUpdateInfoFile(h.obs, h.updateInfo, h.log, &b, date, hotPatchUpdateInfoDir) + service.UploadUpdateInfoFile(h.obs, h.updateInfo, h.log, &b, date, + hotPatchUpdateInfoDir, hotPatchEarlyUpdateInfoDir) } } diff --git a/cve-vulner-manager/cve-ddd/domain/service/service.go b/cve-vulner-manager/cve-ddd/domain/service/service.go index c56af68..47bef77 100644 --- a/cve-vulner-manager/cve-ddd/domain/service/service.go +++ b/cve-vulner-manager/cve-ddd/domain/service/service.go @@ -11,13 +11,13 @@ import ( // UploadUpdateInfoFile upload updateinfo file func UploadUpdateInfoFile(obs obs.OBS, updateinfo updateinfo.UpdateInfo, - log *logrus.Entry, bulletin *domain.SecurityBulletin, date, UpdateInfoDir string, + log *logrus.Entry, bulletin *domain.SecurityBulletin, date, DownloadDir, UploadDir string, ) { for _, branch := range bulletin.AffectedVersion { - filePath := UpdateInfoDir + branch + "/updateinfo.xml" - downloadBys, err := obs.Download(filePath) + downloadPath := DownloadDir + branch + "/updateinfo.xml" + downloadBys, err := obs.Download(downloadPath) if err != nil { - log.Errorf("download %s failed: %s", filePath, err.Error()) + log.Errorf("download %s failed: %s", downloadPath, err.Error()) continue } @@ -33,8 +33,9 @@ func UploadUpdateInfoFile(obs obs.OBS, updateinfo updateinfo.UpdateInfo, continue } - if err = obs.Upload(filePath, data); err != nil { - log.Errorf("upload %s failed: %s", filePath, err.Error()) + uploadPath := UploadDir + branch + "/updateinfo.xml" + if err = obs.Upload(uploadPath, data); err != nil { + log.Errorf("upload %s failed: %s", uploadPath, err.Error()) continue } } -- Gitee From 7a29516e2b1bac0a0878d9276d87dc40bfbc5c1d Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Fri, 13 Dec 2024 16:20:53 +0800 Subject: [PATCH 3/9] revert dir --- cve-vulner-manager/cve-ddd/app/bulletin.go | 2 +- cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index 7b9dab9..0622140 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -152,7 +152,7 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin updateFixedFiles = append(updateFixedFiles, v.PathAppendToIndexFile()) - service.UploadUpdateInfoFile(b.obs, b.updateinfo, b.log, &v, date, updateInfoRootDir, updateInfoEarlyDir) + service.UploadUpdateInfoFile(b.obs, b.updateinfo, b.log, &v, date, updateInfoEarlyDir, updateInfoEarlyDir) } b.uploadIndexAndFixed(uploadDir, indexContent, updateFixedFiles) diff --git a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go index a196c1c..8cd5e32 100644 --- a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go @@ -132,7 +132,7 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir, date string) erro cvesForUpdateInfo = append(cvesForUpdateInfo, b.Cves...) service.UploadUpdateInfoFile(h.obs, h.updateInfo, h.log, &b, date, - hotPatchUpdateInfoDir, hotPatchEarlyUpdateInfoDir) + hotPatchEarlyUpdateInfoDir, hotPatchEarlyUpdateInfoDir) } } -- Gitee From 5e9552ec1a48e38b2c4acea3fc9d7e44cffe3d92 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Fri, 13 Dec 2024 15:04:24 +0800 Subject: [PATCH 4/9] ignore mindspore --- cve-vulner-manager/taskhandler/cve.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index ecedbda..014533b 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -927,11 +927,11 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) openGausVersion = append(openGausVersion, ver) gaussFlag = true } - if flag, ids := judgeSocpe(key, ver); flag && len(ids) > 0 { - mindSporeids = append(mindSporeids, ids...) - mindSporeVersion = append(mindSporeVersion, ver) - sporeFlag = true - } + //if flag, ids := judgeSocpe(key, ver); flag && len(ids) > 0 { + // mindSporeids = append(mindSporeids, ids...) + // mindSporeVersion = append(mindSporeVersion, ver) + // sporeFlag = true + //} olky := models.OpenLookengYaml{PackageName: key, Version: ver} looKengErr := models.GetOpenLookengYaml(&olky, "PackageName", "Version") if looKengErr == nil && olky.Id > 0 { -- Gitee From ce679c493cd300ca441231d5aa03448c16c2457c Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Fri, 13 Dec 2024 15:42:25 +0800 Subject: [PATCH 5/9] forward to ng --- cve-vulner-manager/controllers/upload.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/cve-vulner-manager/controllers/upload.go b/cve-vulner-manager/controllers/upload.go index bf65606..e5848e6 100644 --- a/cve-vulner-manager/controllers/upload.go +++ b/cve-vulner-manager/controllers/upload.go @@ -1,10 +1,14 @@ package controllers import ( + "bytes" "encoding/json" "fmt" + "net/http" "strings" + "github.com/opensourceways/server-common-lib/utils" + "cvevulner/common" "cvevulner/errcode" "cvevulner/models" @@ -137,6 +141,20 @@ func (u *CveErrorFeedBackController) Get() { } } +func forwardToCveManagerNG(body []byte) { + ngUrl := "https://cvemanager-ng.osinfra.cn/api/v1/cve/upload" + req, err := http.NewRequest(http.MethodPost, ngUrl, bytes.NewBuffer(body)) + if err != nil { + logs.Error("new request when forward to ng failed: ", err) + return + } + + client := utils.NewHttpClient(3) + if _, err = client.ForwardTo(req, nil); err != nil { + logs.Error("forward to ng failed: ", err) + } +} + // @Title UserUpload // @Description UserUpload // @Param body body models.OriginUpstream true "body for user content" @@ -144,6 +162,7 @@ func (u *CveErrorFeedBackController) Get() { // @Failure 403 body is empty // @router / [post] func (u *UserUploadController) Post() { + forwardToCveManagerNG(u.Ctx.Input.RequestBody) var uploaddata common.UploadData var ResDataList []ResultData req := u.Ctx.Request -- Gitee From 5f217e62e1f1cd77d197e93db9697e5e5d3b0278 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Mon, 16 Dec 2024 18:50:19 +0800 Subject: [PATCH 6/9] use internal api --- cve-vulner-manager/controllers/upload.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cve-vulner-manager/controllers/upload.go b/cve-vulner-manager/controllers/upload.go index e5848e6..6c806b1 100644 --- a/cve-vulner-manager/controllers/upload.go +++ b/cve-vulner-manager/controllers/upload.go @@ -142,7 +142,7 @@ func (u *CveErrorFeedBackController) Get() { } func forwardToCveManagerNG(body []byte) { - ngUrl := "https://cvemanager-ng.osinfra.cn/api/v1/cve/upload" + ngUrl := "http://cve-manager-ng-service.cve-manager-ng:8080/api/v1/cve/upload" req, err := http.NewRequest(http.MethodPost, ngUrl, bytes.NewBuffer(body)) if err != nil { logs.Error("new request when forward to ng failed: ", err) -- Gitee From 3434c6d0ff7e833b3c3571bdbbeeabd75523dca3 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Tue, 17 Dec 2024 14:29:56 +0800 Subject: [PATCH 7/9] ignore opengauss --- cve-vulner-manager/taskhandler/cve.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index 014533b..d84512a 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -921,12 +921,12 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) if len(versionList) > 0 { for _, ver := range versionList { if ver != "" && len(ver) > 0 { - opy := models.OpenGussYaml{PackageName: key, Version: ver} - gaussErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") - if gaussErr == nil && opy.Id > 0 { - openGausVersion = append(openGausVersion, ver) - gaussFlag = true - } + //opy := models.OpenGussYaml{PackageName: key, Version: ver} + //gaussErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + //if gaussErr == nil && opy.Id > 0 { + // openGausVersion = append(openGausVersion, ver) + // gaussFlag = true + //} //if flag, ids := judgeSocpe(key, ver); flag && len(ids) > 0 { // mindSporeids = append(mindSporeids, ids...) // mindSporeVersion = append(mindSporeVersion, ver) -- Gitee From 9d4521df4565ff6620e99ac3e83835d345408083 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Thu, 19 Dec 2024 16:35:21 +0800 Subject: [PATCH 8/9] optimize updainfo logic --- cve-vulner-manager/cve-ddd/app/bulletin.go | 9 +++-- .../cve-ddd/app/refactor_hotpatch.go | 11 ++++-- cve-vulner-manager/cve-ddd/domain/config.go | 4 ++ .../cve-ddd/domain/service/service.go | 38 +++++++++++++++---- 4 files changed, 47 insertions(+), 15 deletions(-) diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index 0622140..ef2f2c8 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -31,8 +31,8 @@ const ( fileIndex = "index.txt" fileUpdateFixed = "update_fixed.txt" - updateInfoEarlyDir = "earlyupdateinfo/" - updateInfoRootDir = "updateinfo/" + updateInfoEarlyDir = "earlyupdateinfo" + updateInfoRootDir = "updateinfo" ) type BulletinService interface { @@ -131,6 +131,9 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin var updateFixedFiles []string + // 将正式目录的updateinfo文件同步到early目录，这样每次提前生成的文件都是以正式目录为基准的 + service.SyncUpdateInfoToEarlyDir(b.obs, b.log, handleBranch, updateInfoRootDir, updateInfoEarlyDir) + for _, v := range bulletins { maxColdPatchId++ @@ -152,7 +155,7 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin updateFixedFiles = append(updateFixedFiles, v.PathAppendToIndexFile()) - service.UploadUpdateInfoFile(b.obs, b.updateinfo, b.log, &v, date, updateInfoEarlyDir, updateInfoEarlyDir) + service.UploadUpdateInfoFile(b.obs, b.updateinfo, b.log, &v, date, updateInfoEarlyDir) } b.uploadIndexAndFixed(uploadDir, indexContent, updateFixedFiles) diff --git a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go index 8cd5e32..83da28a 100644 --- a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go @@ -20,8 +20,8 @@ import ( const ( fileHotPatch = "update_hot_patch.txt" - hotPatchUpdateInfoDir = "updateinfo-hotpatch/" - hotPatchEarlyUpdateInfoDir = "hotpatch-early-updateinfo/" + hotPatchUpdateInfoDir = "updateinfo-hotpatch" + hotPatchEarlyUpdateInfoDir = "hotpatch-early-updateinfo" ) type RefactorHotPatchService interface { @@ -107,6 +107,10 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir, date string) erro cves[k].Component = issue.Component } + // 将正式目录的updateinfo文件同步到early目录，这样每次提前生成的文件都是以正式目录为基准的 + service.SyncUpdateInfoToEarlyDir(h.obs, h.log, domain.GetMaintainVersion(), + hotPatchUpdateInfoDir, hotPatchEarlyUpdateInfoDir) + bulletins := cves.GenerateBulletins() for _, b := range bulletins { b.PatchUrl = issue.PatchUrl @@ -131,8 +135,7 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir, date string) erro cvesForUpdateInfo = append(cvesForUpdateInfo, b.Cves...) - service.UploadUpdateInfoFile(h.obs, h.updateInfo, h.log, &b, date, - hotPatchEarlyUpdateInfoDir, hotPatchEarlyUpdateInfoDir) + service.UploadUpdateInfoFile(h.obs, h.updateInfo, h.log, &b, date, hotPatchEarlyUpdateInfoDir) } } diff --git a/cve-vulner-manager/cve-ddd/domain/config.go b/cve-vulner-manager/cve-ddd/domain/config.go index f44aeea..889d9d3 100644 --- a/cve-vulner-manager/cve-ddd/domain/config.go +++ b/cve-vulner-manager/cve-ddd/domain/config.go @@ -7,3 +7,7 @@ var maintainVersion sets.Set[string] func InitMaintainVersion(version []string) { maintainVersion = sets.New(version...) } + +func GetMaintainVersion() []string { + return maintainVersion.UnsortedList() +} diff --git a/cve-vulner-manager/cve-ddd/domain/service/service.go b/cve-vulner-manager/cve-ddd/domain/service/service.go index 47bef77..2f02c2f 100644 --- a/cve-vulner-manager/cve-ddd/domain/service/service.go +++ b/cve-vulner-manager/cve-ddd/domain/service/service.go @@ -2,6 +2,8 @@ package service import ( + "fmt" + "github.com/sirupsen/logrus" "cvevulner/cve-ddd/domain" @@ -9,15 +11,36 @@ import ( "cvevulner/cve-ddd/domain/updateinfo" ) -// UploadUpdateInfoFile upload updateinfo file +const ( + updateInfoFileName = "updateinfo.xml" +) + +// SyncUpdateInfoToEarlyDir 将正式目录的updateinfo文件同步到early目录，这样每次提前生成的文件都是以正式目录为基准的 +func SyncUpdateInfoToEarlyDir(obs obs.OBS, log *logrus.Entry, branches []string, officialDir, earlyDir string) { + for _, b := range branches { + officialPath := fmt.Sprintf("%s/%s/%s", officialDir, b, updateInfoFileName) + downloadBys, err := obs.Download(officialPath) + if err != nil { + log.Errorf("sync download %s failed: %s", officialPath, err.Error()) + continue + } + + earlyPath := fmt.Sprintf("%s/%s/%s", earlyDir, b, updateInfoFileName) + if err = obs.Upload(earlyPath, downloadBys); err != nil { + log.Errorf("sync upload %s failed: %s", earlyPath, err.Error()) + } + } +} + +// UploadUpdateInfoFile 用以以前生成updateinfo文件，以便测试验证文件正确性 func UploadUpdateInfoFile(obs obs.OBS, updateinfo updateinfo.UpdateInfo, - log *logrus.Entry, bulletin *domain.SecurityBulletin, date, DownloadDir, UploadDir string, + log *logrus.Entry, bulletin *domain.SecurityBulletin, date, earlyDir string, ) { for _, branch := range bulletin.AffectedVersion { - downloadPath := DownloadDir + branch + "/updateinfo.xml" - downloadBys, err := obs.Download(downloadPath) + filePath := fmt.Sprintf("%s/%s/%s", earlyDir, branch, updateInfoFileName) + downloadBys, err := obs.Download(filePath) if err != nil { - log.Errorf("download %s failed: %s", downloadPath, err.Error()) + log.Errorf("download %s failed: %s", filePath, err.Error()) continue } @@ -33,9 +56,8 @@ func UploadUpdateInfoFile(obs obs.OBS, updateinfo updateinfo.UpdateInfo, continue } - uploadPath := UploadDir + branch + "/updateinfo.xml" - if err = obs.Upload(uploadPath, data); err != nil { - log.Errorf("upload %s failed: %s", uploadPath, err.Error()) + if err = obs.Upload(filePath, data); err != nil { + log.Errorf("upload %s failed: %s", filePath, err.Error()) continue } } -- Gitee From f155166c9b1870e4c822e85740cefddf3d2507b5 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Mon, 23 Dec 2024 09:41:58 +0800 Subject: [PATCH 9/9] ignore mindspore and opengauss --- cve-vulner-manager/taskhandler/cve.go | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index d84512a..a89451e 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -921,17 +921,6 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) if len(versionList) > 0 { for _, ver := range versionList { if ver != "" && len(ver) > 0 { - //opy := models.OpenGussYaml{PackageName: key, Version: ver} - //gaussErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") - //if gaussErr == nil && opy.Id > 0 { - // openGausVersion = append(openGausVersion, ver) - // gaussFlag = true - //} - //if flag, ids := judgeSocpe(key, ver); flag && len(ids) > 0 { - // mindSporeids = append(mindSporeids, ids...) - // mindSporeVersion = append(mindSporeVersion, ver) - // sporeFlag = true - //} olky := models.OpenLookengYaml{PackageName: key, Version: ver} looKengErr := models.GetOpenLookengYaml(&olky, "PackageName", "Version") if looKengErr == nil && olky.Id > 0 { -- Gitee