diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index f5e401ef409774e5fc31f52d28ff6080df423fbb..801e5ae92f272e54bc219814cf7a8519d15a3061 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -180,7 +180,7 @@ cve_number_t = 2018 # Create an issue's repo whitelist;1: open; 2: close issue_whitelist = 2 # List of affected branches: openEuler-20.03-LTS,openEuler-20.03-LTS-SP1 -affected_branchs = "openEuler-20.03-LTS-SP4,openEuler-22.03-LTS-SP1,openEuler-22.03-LTS-SP3,openEuler-22.03-LTS-SP4,master,openEuler-24.03-LTS,openEuler-24.03-LTS-SP1,openEuler-24.03-LTS-Next" +affected_branchs = "openEuler-20.03-LTS-SP4,openEuler-22.03-LTS-SP3,openEuler-22.03-LTS-SP4,master,openEuler-24.03-LTS,openEuler-24.03-LTS-SP1,openEuler-24.03-LTS-Next" abandoned_branchs = "openEuler-20.03-LTS,openEuler-21.03,openEuler-21.09,openEuler-20.09" # Close the highest privilege of issue:1:open;2:close close_issue_privilege = 2 @@ -226,7 +226,7 @@ snsuffix = 1002 # openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv # public sa format: openEuler-20.03-LTS@https;openEuler-20.03-LTS-SP1@https v_pack_20_03_url = "openEuler-20.03-LTS-SP4@https;openEuler-22.03-LTS-SP1@https;openEuler-22.03-LTS-SP3@https;openEuler-22.03-LTS-SP4@https;openEuler-24.03-LTS@https" -release_date_of_version = "openEuler-20.03-LTS-SP4:2023-12-12;openEuler-22.03-LTS-SP3:2024-01-01;openEuler-24.03-LTS:2024-05-30;openEuler-22.03-LTS-SP4:2024-06-29" +release_date_of_version = "openEuler-20.03-LTS-SP4:2023-12-12;openEuler-22.03-LTS-SP3:2024-01-01;openEuler-24.03-LTS:2024-05-30;openEuler-22.03-LTS-SP4:2024-06-29;openEuler-24.03-LTS-SP1:2024-12-22" # Time difference in different time zones sa_timestamp_zone = 28810 unaffect_year = 2018 diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index ef2f2c8d93ac232a14bdf362c57f012cc7fbf983..11c1f0000c59b503d6d804373a7d49100e6d408e 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -2,8 +2,10 @@ package app import ( "encoding/json" + "errors" "fmt" "net/http" + "path/filepath" "strings" "sync" "time" @@ -37,6 +39,7 @@ const ( type BulletinService interface { GenerateBulletins([]string, string) (string, error) + Move() error } func NewBulletinService( @@ -312,3 +315,48 @@ func (b *bulletinService) initReleaseDate() { b.releaseDate.Store(key, value.AddDate(0, 0, 1)) } } + +func (b *bulletinService) Move() error { + fromDir := beego.AppConfig.String("obs::upload_cvrf_dir") + toDir := beego.AppConfig.String("obs::download_cvrf_dir") + + // 查询当天产生的目录 + prefix := fromDir + time.Now().Format("2006-01-02") + latestDir, err := b.obs.GetLatestDirWithPrefix(prefix) + if err != nil { + return err + } + + if latestDir == "" { + return errors.New("can not find today's dir") + } + + files, err := b.obs.ListObjects(latestDir) + if err != nil { + return err + } + + for _, f := range files { + content, err := b.obs.Download(f) + if err != nil { + b.log.Errorf("move file: download %s failed: %s", f, err.Error()) + continue + } + + var uploadPath string + fileName := filepath.Base(f) + if fileName == fileIndex || fileName == fileUpdateFixed { + uploadPath = toDir + fileName + } else { + uploadPath = fmt.Sprintf("%s%d/%s", toDir, time.Now().Year(), fileName) + } + + if err = b.obs.Upload(uploadPath, content); err != nil { + b.log.Errorf("move file: upload %s failed: %s", uploadPath, err.Error()) + } else { + b.log.Infof("move file: upload success: %s", uploadPath) + } + } + + return nil +} diff --git a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go index 83da28a02ea3f404c91dd1bc28c4ec5e24311181..f2512f935dd78da8dc557818c10dd81c11b923cc 100644 --- a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go @@ -18,8 +18,6 @@ import ( ) const ( - fileHotPatch = "update_hot_patch.txt" - hotPatchUpdateInfoDir = "updateinfo-hotpatch" hotPatchEarlyUpdateInfoDir = "hotpatch-early-updateinfo" ) @@ -66,11 +64,6 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir, date string) erro var cvesForUpdateInfo domain.Cves var uploadFileName []string - indexContent, err := h.getIndexContent(uploadDir) - if err != nil { - return fmt.Errorf("get %s failed: %w", fileIndex, err) - } - maxHotPatchId, err := h.hotPatch.MaxHotPatchID() if err != nil { return fmt.Errorf("parse max id failed: %w", err) @@ -143,7 +136,7 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir, date string) erro return fmt.Errorf("no new hot patch issues") } - h.uploadIndexAndHotPatch(uploadDir, indexContent, uploadFileName) + h.uploadIndexAndFixed(uploadDir, uploadFileName) return h.uploadUpdateInfo(cvesForUpdateInfo) } @@ -157,19 +150,30 @@ func (h *refactorHotPatchService) getIndexContent(uploadDir string) (string, err return string(content), err } -func (h *refactorHotPatchService) uploadIndexAndHotPatch(uploadDir, indexContent string, hotPatchFiles []string) { - updateHotPatchContent := strings.TrimSpace(strings.Join(hotPatchFiles, EOF)) - newIndexContent := strings.TrimSpace(indexContent) + EOF + updateHotPatchContent - +func (h *refactorHotPatchService) uploadIndexAndFixed(uploadDir string, hotPatchFiles []string) { indexPath := uploadDir + fileIndex - updateFixedPath := uploadDir + fileHotPatch + indexContent, err := h.obs.Download(indexPath) + if err != nil { + logrus.Errorf("download %s failed: %s", fileIndex, err.Error()) + return + } - if err := h.obs.Upload(indexPath, []byte(newIndexContent)); err != nil { + updateHotPatchContent := strings.TrimSpace(strings.Join(hotPatchFiles, EOF)) + newIndexContent := strings.TrimSpace(string(indexContent)) + EOF + updateHotPatchContent + if err = h.obs.Upload(indexPath, []byte(newIndexContent)); err != nil { h.log.Errorf("upload %s failed: %v", fileIndex, err) } - if err := h.obs.Upload(updateFixedPath, []byte(updateHotPatchContent)); err != nil { - h.log.Errorf("upload %s failed: %v", fileHotPatch, err) + updateFixedPath := uploadDir + fileUpdateFixed + fixedContent, err := h.obs.Download(updateFixedPath) + if err != nil { + logrus.Errorf("download %s failed: %s", fileUpdateFixed, err.Error()) + return + } + + newFixedContent := strings.TrimSpace(string(fixedContent)) + EOF + updateHotPatchContent + if err = h.obs.Upload(updateFixedPath, []byte(newFixedContent)); err != nil { + h.log.Errorf("upload %s failed: %v", fileUpdateFixed, err) } } diff --git a/cve-vulner-manager/cve-ddd/controller/cve.go b/cve-vulner-manager/cve-ddd/controller/cve.go index f48a68a98e13f6ef13fd3c8d2b149cf8b42d0b6b..c3e666864562139832c56c9ea4da5f4a09fbcecd 100644 --- a/cve-vulner-manager/cve-ddd/controller/cve.go +++ b/cve-vulner-manager/cve-ddd/controller/cve.go @@ -107,3 +107,12 @@ func (c *CveController) Generate() { c.success(nil) } + +// Move notice file from upload dir to release dir +func (c *CveController) Move() { + if err := c.BulletinService.Move(); err != nil { + c.fail(err.Error()) + } else { + c.success(nil) + } +} diff --git a/cve-vulner-manager/cve-ddd/domain/obs/obs.go b/cve-vulner-manager/cve-ddd/domain/obs/obs.go index e3c6c960dc5d515c7e9016926d34ccd8eba034db..f6c34e569c33d5dac8d9ff75d6a112e122c29acb 100644 --- a/cve-vulner-manager/cve-ddd/domain/obs/obs.go +++ b/cve-vulner-manager/cve-ddd/domain/obs/obs.go @@ -1,10 +1,8 @@ package obs type OBS interface { - UploadToDynamicDir(fileName string, data []byte) error - DownloadFromDynamicDir(fileName string) ([]byte, error) - UploadUpdateInfo(fileName string, data []byte) error - Upload(path string, data []byte) error Download(path string) ([]byte, error) + GetLatestDirWithPrefix(prefix string) (string, error) + ListObjects(prefix string) ([]string, error) } diff --git a/cve-vulner-manager/cve-ddd/infrastructure/latestrpmimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/latestrpmimpl/impl.go index 17aee9879eb9bf0877e934a00abb23625eab9673..21b8fdca89f7e948dd1057dd03bf99403ef13070 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/latestrpmimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/latestrpmimpl/impl.go @@ -12,6 +12,7 @@ import ( "github.com/astaxie/beego" "github.com/opensourceways/robot-gitee-lib/client" + "github.com/sirupsen/logrus" ) type PkgRPM struct { @@ -52,12 +53,14 @@ func (l *latestRpm) InitData(branches []string) error { path := fmt.Sprintf("%s%s.csv", l.rpm.PathPrefix, branch) content, err := l.cli.GetPathContent(l.rpm.Org, l.rpm.Repo, path, l.rpm.Branch) if err != nil { - return err + logrus.Errorf("new-cold-patch-cve-collect init Data of %s failed: %s", path, err.Error()) + continue } decodeContent, err := base64.StdEncoding.DecodeString(content.Content) if err != nil { - return err + logrus.Errorf("new-cold-patch-cve-collect decode Data of %s failed: %s", path, err.Error()) + continue } l.buildTime[branch] = l.parseFile(decodeContent) diff --git a/cve-vulner-manager/cve-ddd/infrastructure/obsimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/obsimpl/impl.go index 856306ddab2a0ba02330d62d9375427bd43864e7..c384204061fa6257c2acb67b95b708a883a22c33 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/obsimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/obsimpl/impl.go @@ -2,11 +2,7 @@ package obsimpl import ( "bytes" - "fmt" "io" - "io/ioutil" - "strings" - "time" "github.com/astaxie/beego" "github.com/huaweicloud/huaweicloud-sdk-go-obs/obs" @@ -52,10 +48,10 @@ type obsImpl struct { cli *obs.ObsClient } -func (impl obsImpl) UploadToDynamicDir(fileName string, data []byte) error { +func (impl obsImpl) Upload(path string, data []byte) error { input := &obs.PutObjectInput{} input.Bucket = impl.cfg.Bucket - input.Key = impl.getDynamicDir() + fileName + input.Key = path input.Body = bytes.NewReader(data) _, err := impl.cli.PutObject(input) @@ -63,60 +59,57 @@ func (impl obsImpl) UploadToDynamicDir(fileName string, data []byte) error { return err } -func (impl obsImpl) DownloadFromDynamicDir(fileName string) ([]byte, error) { +func (impl obsImpl) Download(path string) ([]byte, error) { input := &obs.GetObjectInput{} input.Bucket = impl.cfg.Bucket - input.Key = impl.getDynamicDir() + fileName + input.Key = path output, err := impl.cli.GetObject(input) if err != nil { return nil, err } defer output.Body.Close() - return ioutil.ReadAll(output.Body) + return io.ReadAll(output.Body) } -func (impl obsImpl) UploadUpdateInfo(fileName string, data []byte) error { - input := &obs.PutObjectInput{} - input.Bucket = impl.cfg.Bucket - nowStr := time.Now().Format("2006-01-02") - input.Key = fmt.Sprintf("%s%s-%s/%s", impl.cfg.UpdateInfoDir, nowStr, "hotpatch", fileName) - input.Body = bytes.NewReader(data) - - _, err := impl.cli.PutObject(input) +func (impl obsImpl) GetLatestDirWithPrefix(prefix string) (string, error) { + input := obs.ListObjectsInput{ + Bucket: impl.cfg.Bucket, + ListObjsInput: obs.ListObjsInput{ + Prefix: prefix, + Delimiter: "/", + }, + } - return err -} + out, err := impl.cli.ListObjects(&input) + if err != nil { + return "", err + } -func (impl obsImpl) getDynamicDir() string { - todayStr := time.Now().Format("2006-01-02") - if strings.Contains(dynamicDir, todayStr) { - return dynamicDir + if len(out.CommonPrefixes) == 0 { + return "", nil } - return fmt.Sprintf("%s%s-%s/", impl.cfg.UpdateInfoDir, todayStr, "hotpatch") + return out.CommonPrefixes[len(out.CommonPrefixes)-1], nil } -func (impl obsImpl) Upload(path string, data []byte) error { - input := &obs.PutObjectInput{} - input.Bucket = impl.cfg.Bucket - input.Key = path - input.Body = bytes.NewReader(data) - - _, err := impl.cli.PutObject(input) - - return err -} +func (impl obsImpl) ListObjects(prefix string) ([]string, error) { + input := obs.ListObjectsInput{ + Bucket: impl.cfg.Bucket, + ListObjsInput: obs.ListObjsInput{ + Prefix: prefix, + }, + } -func (impl obsImpl) Download(path string) ([]byte, error) { - input := &obs.GetObjectInput{} - input.Bucket = impl.cfg.Bucket - input.Key = path - output, err := impl.cli.GetObject(input) + out, err := impl.cli.ListObjects(&input) if err != nil { return nil, err } - defer output.Body.Close() - return io.ReadAll(output.Body) + var list []string + for _, v := range out.Contents { + list = append(list, v.Key) + } + + return list, nil } diff --git a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go index ba4e1ae9262c538fcaa08421a7dc9b693b18419c..8b843bca9a3cbd99a05d5a096572129827706382 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go @@ -152,6 +152,10 @@ func (impl updateInfoImpl) updateXml(sb *domain.SecurityBulletin, branch, date s } for _, productPackage := range pl { + if productPackage.CPE != branch { + continue + } + var pe Package pe.Filename = productPackage.FullName packVersionList := strings.Split(productPackage.FullName, "-") diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go index c9f0a1a98979a636946343ad25da7c223edd26b0..b42e3d9b0740ea64c34bbd0e62560bfcc5a0bf84 100644 --- a/cve-vulner-manager/models/issue.go +++ b/cve-vulner-manager/models/issue.go @@ -789,3 +789,22 @@ func (t *IssueTemplate) IsIssueComplete() bool { const StatusCompleted = 3 return t.Status == StatusCompleted } + +// HasAffected checks if the issue has affected versions based on the AnalysisVersion field. +func (t *IssueTemplate) HasAffected() bool { + const ExpectedItemLength = 2 + split := strings.Split(t.AnalysisVersion, ",") + for _, v := range split { + item := strings.Split(v, ":") + if len(item) != ExpectedItemLength || item[1] == "" { + continue + } + + _, ok := common.AnalysisUnaffected[item[1]] + if !ok { + return true + } + } + + return false +} diff --git a/cve-vulner-manager/routers/new_router.go b/cve-vulner-manager/routers/new_router.go index ff6cace0308f176c0371e1c7299cbb43eccf703a..5bcfc0675209618445395e9e8d2f3a15fe8514b0 100644 --- a/cve-vulner-manager/routers/new_router.go +++ b/cve-vulner-manager/routers/new_router.go @@ -84,4 +84,5 @@ func initNewRouter() { beego.Router("/security/bulletin/collect", NewCveController, "post:CollectCveData") beego.Router("/security/bulletin/generate", NewCveController, "post:Generate") + beego.Router("/security/bulletin/move", NewCveController, "post:Move") } diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go index 2fc1021f1f9b373042d9b6b9af72457ad6c472cb..1c94debd319376ed5d6e1faabc3e3dce0f32439d 100644 --- a/cve-vulner-manager/task/issuetask.go +++ b/cve-vulner-manager/task/issuetask.go @@ -582,8 +582,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) er logs.Info("ProcUpdateIssue, Successfully updated the issue template, "+ "CveNum: ", issueValue.CveNum, ", templetID: ", templetID) // Judgment of necessary fields - if (it.NVDScore <= 0 && it.OwnedComponent != "kernel") || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || - len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + if issueValue.CveNum == "" || issueValue.PackName == "" || issueValue.CveVersion == "" { logs.Error("ProcUpdateIssue, Field is empty: NVDScore: ", it.NVDScore, ",Description: ", issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) diff --git a/cve-vulner-manager/taskhandler/excel.go b/cve-vulner-manager/taskhandler/excel.go index 830c9369ae2e6704b29ccbfedf4ae13e3a0c5843..5f17dce3fd36c251a3d44f6ca8957d7327c9f4eb 100644 --- a/cve-vulner-manager/taskhandler/excel.go +++ b/cve-vulner-manager/taskhandler/excel.go @@ -1188,6 +1188,18 @@ func UnaffectIssueProc(affectBranch string, cvrfFileList map[string][]string, continue } + // 新数据,检查openeuler评分和向量 + if v.IsIssueWithAnalysisVersion() && v.HasAffected() { + commentFunc := func(content string) { + AddCommentToIssue(content, v.IssueNum, owner, v.Repo, accessToken) + } + + if !CheckOpenEulerScoreAndVector(&v, commentFunc) { + logs.Error("CheckOpenEulerScoreAndVector of [%s %s] failed", v.Repo, v.IssueNum) + continue + } + } + var status string var released bool issueExist, _ := GetCveSecurityNotice(v.CveNum, v.Repo, true) @@ -1407,6 +1419,29 @@ func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, return } +// CheckOpenEulerScoreAndVector checks if the OpenEulerScore and OpenEulerVector fields of the issue are set. +// If either field is not set, it sends a comment to the issue and returns false. +func CheckOpenEulerScoreAndVector(issue *models.IssueTemplate, comment func(content string)) bool { + if issue.OpenEulerScore == 0 || issue.OpenEulerVector == "" { + cc := fmt.Sprintf("@%v CVSS评分和矢量值不能为空", issue.Assignee) + comment(cc) + + return false + } + + score, _ := models.QueryIssueScore(issue.CveId) + if score.ScoreType == "V3" { + if util.CalculateCVSSV3BaseScore(issue.OpenEulerVector) != issue.OpenEulerScore { + cc := fmt.Sprintf("@%v CVSS评分和矢量值不一致,请修改至矢量值和CVSS评分一致!", issue.Assignee) + comment(cc) + + return false + } + } + + return true +} + func InitReleaseDate() { releaseDate = make(map[string]int64) releaseDateConfig := beego.AppConfig.DefaultString("excel::release_date_of_version", "") diff --git a/cve-vulner-manager/taskhandler/issuestatistics.go b/cve-vulner-manager/taskhandler/issuestatistics.go index a6f29de41805e3f1e3a5b60e52793bcb24a7871c..66a82f0823e1f895f0106d23021f15face024045 100644 --- a/cve-vulner-manager/taskhandler/issuestatistics.go +++ b/cve-vulner-manager/taskhandler/issuestatistics.go @@ -326,7 +326,7 @@ func ProcSecLinkTemplate(beforeDate, prcnum int, owner, accessToken string) erro } // Determine whether cve has been processed exist, saData := GetCveSecurityNotice(temp.CveNum, temp.Repo, true) - if exist && saData.Result.IsFixed() && len(saData.Result.AffectedProduct) > 2 { + if exist && saData.Result.AffectedProduct != "" { // Update sa release time UpdateSAReleaseTime(saData.Result.AffectedProduct, saData.Result.CreateTime, temp.TemplateId) secLink := secLinkConfig + "/zh/security/safety-bulletin/detail/?id=" + saData.Result.AffectedProduct