diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 3f6af3511f3395f4dbf47772010a389e70af2b98..52dfc21b3370f0949c1286a2bab5b407d34d8145 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -224,8 +224,8 @@ snsuffix = 1002 # example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; # openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv # public sa format: openEuler-20.03-LTS@https;openEuler-20.03-LTS-SP1@https -v_pack_20_03_url = "openEuler-20.03-LTS-SP4@https;openEuler-22.03-LTS-SP1@https;openEuler-22.03-LTS-SP3@https;openEuler-24.03-LTS@https" -release_date_of_version = "openEuler-20.03-LTS-SP4:2023-12-12;openEuler-22.03-LTS-SP3:2024-01-01;openEuler-24.03-LTS:2024-05-30" +v_pack_20_03_url = "openEuler-20.03-LTS-SP4@https;openEuler-22.03-LTS-SP1@https;openEuler-22.03-LTS-SP3@https;openEuler-22.03-LTS-SP4@https;openEuler-24.03-LTS@https" +release_date_of_version = "openEuler-20.03-LTS-SP4:2023-12-12;openEuler-22.03-LTS-SP3:2024-01-01;openEuler-24.03-LTS:2024-05-30;openEuler-22.03-LTS-SP4:2024-06-29" # Time difference in different time zones sa_timestamp_zone = 28810 unaffect_year = 2018 diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index 5eb0ab35c461fe7de6239655cf7c6d3943cf473c..d9bcc106bfc58f0e6d50dd405ca2190aac556a21 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -5,6 +5,7 @@ import ( "fmt" "net/http" "strings" + "sync" "time" "github.com/astaxie/beego" @@ -14,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "cvevulner/cve-ddd/domain" + "cvevulner/cve-ddd/domain/backend" "cvevulner/cve-ddd/domain/bulletin" "cvevulner/cve-ddd/domain/majun" "cvevulner/cve-ddd/domain/obs" @@ -22,7 +24,7 @@ import ( ) const ( - EOF = "\r\n" + EOF = "\n" fileIndex = "index.txt" fileUpdateFixed = "update_fixed.txt" @@ -38,26 +40,34 @@ func NewBulletinService( m majun.Majun, b bulletin.Bulletin, t testresult.Result, + bd backend.Backend, l *logrus.Entry, ) *bulletinService { - return &bulletinService{ + service := &bulletinService{ obs: o, repo: r, maJun: m, bulletin: b, testResult: t, + backend: bd, log: l, giteeToken: beego.AppConfig.String("gitee::git_token"), } + + service.initReleaseDate() + + return service } type bulletinService struct { - obs obs.OBS - repo repository.CveRepository - maJun majun.Majun - bulletin bulletin.Bulletin - testResult testresult.Result - giteeToken string + obs obs.OBS + repo repository.CveRepository + maJun majun.Majun + bulletin bulletin.Bulletin + testResult testresult.Result + backend backend.Backend + releaseDate sync.Map + giteeToken string log *logrus.Entry } @@ -68,8 +78,11 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin return "", fmt.Errorf("get release branch err: %w", err) } + b.log.Infof("handle branch are %v", handleBranch) + domain.InitMaintainVersion(handleBranch) + b.log.Infof("need to handle cve are %v", cveNum) cves, err := b.repo.FindCves(repository.Option{CveNum: cveNum}) if err != nil { return "", fmt.Errorf("find cves failed: %w", err) @@ -79,9 +92,16 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin return "", fmt.Errorf("no cves") } + if len(cveNum) != len(cves) { + b.log.Errorf("num of cveNum %d and num of cves %d does not match", len(cveNum), len(cves)) + } + // 用关联pr并且合入的分支进行过滤 cves = b.filterByRelatedPR(cves, handleBranch) + // 用已发布的分支进行过滤 + cves = b.filterByPublishedBranch(cves) + b.testResult.Init(handleBranch, date) // 用成功转测的数据进行过滤，只发布转测成功的cve @@ -178,14 +198,41 @@ func (b *bulletinService) filterByRelatedPR(cves domain.Cves, handleBranch []str continue } - if pr.State == "merged" { - relatedPrSets.Insert(pr.Base.Ref) + if pr.State != "merged" { + continue + } + + branch := pr.Base.Ref + + releaseTimeOfBranch, ok := b.releaseDate.Load(branch) + if ok { + releaseTime := releaseTimeOfBranch.(time.Time) + mergeAt, err1 := time.ParseInLocation("2006-01-02T15:04:05+08:00", pr.MergedAt, time.Local) + if err1 != nil { + b.log.Errorf("parse %s %s %s merge time failed: %s", + v.CveNum, v.ColdIssue.Number, branch, err1.Error(), + ) + continue + } + + if releaseTime.After(mergeAt) { + b.log.Errorf("release branch time of [%s %s %s] check failed: merge time: %v, release time: %v", + v.CveNum, v.ColdIssue.Number, branch, mergeAt, releaseTime, + ) + continue + } } + + relatedPrSets.Insert(branch) } intersection := handleBranchSet.Intersection(relatedPrSets) v.AffectedVersion = intersection.UnsortedList() + b.log.Infof("the affected version of [%s %s] after pr filter are %v", + v.CveNum, v.ColdIssue.Number, v.AffectedVersion, + ) + filteredCves = append(filteredCves, v) } @@ -211,3 +258,44 @@ func (b *bulletinService) getRelatedPR(issue domain.Issue) (prs []sdk.PullReques return } + +func (b *bulletinService) filterByPublishedBranch(cves domain.Cves) domain.Cves { + var filteredCves domain.Cves + + for _, v := range cves { + publishedBranch, err := b.backend.PublishedInfo(v.CveNum, v.ColdIssue.Repo) + if err != nil { + b.log.Errorf("filter %s release branch failed: %s", v.CveNum, err.Error()) + continue + } + + publishSets := sets.New(publishedBranch...) + affectedVersion := sets.New(v.AffectedVersion...) + diff := affectedVersion.Difference(publishSets) + + if len(diff) == 0 { + b.log.Errorf("all branch of %s have been published", v.CveNum) + continue + } + + v.AffectedVersion = diff.UnsortedList() + + b.log.Infof("the affected version of [%s %s] after published filter are %v", + v.CveNum, v.ColdIssue.Number, v.AffectedVersion, + ) + + filteredCves = append(filteredCves, v) + } + + return filteredCves +} + +func (b *bulletinService) initReleaseDate() { + releaseDateConfig := beego.AppConfig.DefaultString("excel::release_date_of_version", "") + for _, v := range strings.Split(strings.Trim(releaseDateConfig, ";"), ";") { + split := strings.Split(v, ":") + key := split[0] + value, _ := time.ParseInLocation("2006-01-02", split[1], time.Local) + b.releaseDate.Store(key, value.AddDate(0, 0, 1)) + } +} diff --git a/cve-vulner-manager/cve-ddd/infrastructure/bulletinimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/bulletinimpl/impl.go index 88c2a307b81a3e4eefc454e404e52b02de6fedb5..d534fdc3f6d3ea0c4a29ba85ffd93ebaf7168383 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/bulletinimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/bulletinimpl/impl.go @@ -248,7 +248,7 @@ func (impl bulletinImpl) documentReferences(sb *domain.SecurityBulletin) Documen var cveUrl, other []CveUrl for _, cve := range sb.Cves { - url := impl.cfg.SecurityCveUrlPrefix + fmt.Sprintf("cveId=%s&packageName=%s", cve.CveNum, sb.Component) + url := impl.cfg.SecurityCveUrlPrefix + fmt.Sprintf("cveId=%s", cve.CveNum) cveUrl = append(cveUrl, CveUrl{Url: url}) otherUrl := "https://nvd.nist.gov/vuln/detail/" + cve.CveNum @@ -393,6 +393,11 @@ func (impl bulletinImpl) vulnerability(sb *domain.SecurityBulletin) []Vulnerabil } for k, cve := range sb.Cves { + var productIds []ProductId + for _, v := range sb.AffectedVersion { + productIds = append(productIds, ProductId{ProductId: v}) + } + vul := Vulnerability{ Ordinal: strconv.Itoa(k + 1), Xmlns: xmlns, @@ -410,7 +415,7 @@ func (impl bulletinImpl) vulnerability(sb *domain.SecurityBulletin) []Vulnerabil ProductStatuses: ProductStatuses{ Status: Status{ Type: "Fixed", - ProductId: []ProductId{{ProductId: sb.AffectedVersion[0]}}, + ProductId: productIds, }, }, Threats: Threats{ diff --git a/cve-vulner-manager/cve-ddd/infrastructure/testresultimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/testresultimpl/impl.go index 9c38f0899558e9265758772bcccd40fc8f96d36a..8436835a858da4b342dc10f233d559c93cf66981 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/testresultimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/testresultimpl/impl.go @@ -113,11 +113,15 @@ func (impl *testResultImpl) Filter(cves domain.Cves) domain.Cves { } if len(filteredVersion) == 0 { + impl.log.Errorf("cant find any test repo data of %s", cve.CveNum) + continue // 过滤后的受影响分支为空，整个cve都不用发布公告了 } cve.AffectedVersion = filteredVersion filtered = append(filtered, cve) + + impl.log.Infof("the affected version of %s filtered by test repo are: %v", cve.CveNum, cve.AffectedVersion) } return filtered @@ -127,7 +131,11 @@ func (impl *testResultImpl) GenerateProductTree(component string, affectedVersio tree := make(domain.ProductTree) for _, version := range affectedVersion { - rpms := impl.resultCache[version][component] // 生成公告前已经过滤一次了，所以组件和影响分支必定存在 + rpms, ok := impl.resultCache[version][component] // 生成公告前已经过滤一次了，所以组件和影响分支必定存在 + if !ok { + impl.log.Errorf("cant find product tree of [%s %s]", version, component) + continue + } for _, v := range rpms { // example of rpm: zbar-0.22-4.oe2203.src.rpm diff --git a/cve-vulner-manager/models/common.go b/cve-vulner-manager/models/common.go index af5055c7eb9a559e93a6e4722434f4ab21230cb4..019d435aff4dc8ecb48e722a1bf2a57d5489ae00 100644 --- a/cve-vulner-manager/models/common.go +++ b/cve-vulner-manager/models/common.go @@ -38,8 +38,10 @@ func OpenEulerScoreProc(openEulerScore float64) (CveLevel string) { CveLevel = "High" } else if openEulerScore >= 4.0 && openEulerScore <= 6.9 { CveLevel = "Medium" - } else if openEulerScore <= 3.9 { + } else if openEulerScore >= 0.1 && openEulerScore <= 3.9 { CveLevel = "Low" + } else { + CveLevel = "None" } return CveLevel } diff --git a/cve-vulner-manager/routers/new_router.go b/cve-vulner-manager/routers/new_router.go index eeda1ab582c5f9c053c9d43ef6ba7cd7bb06d019..f1279578ac61fcc29bce03081020d1c5a84f581b 100644 --- a/cve-vulner-manager/routers/new_router.go +++ b/cve-vulner-manager/routers/new_router.go @@ -54,6 +54,7 @@ func InitNewRouter() { majunimpl.NewMajunImpl(), bulletinimpl.NewBulletinImpl(), testresultimpl.NewTestResultImpl(logBulletin), + backendimpl.NewBackendImpl(), logBulletin, ) diff --git a/cve-vulner-manager/taskhandler/cvrf.go b/cve-vulner-manager/taskhandler/cvrf.go index 9aca6aa4ddc49afb8234dcd5628789d35093d5ad..d15d5cbe9ad0ee9d4ddc215124cbb74cf58f36ec 100644 --- a/cve-vulner-manager/taskhandler/cvrf.go +++ b/cve-vulner-manager/taskhandler/cvrf.go @@ -812,7 +812,7 @@ func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, } if !isExist { var cveUrl1 CveUrl - cveUrl1.Url = secLinkConfig + "/en/security/cve/detail.html?id=" + v.CveNum + cveUrl1.Url = secLinkConfig + "/en/security/cve/detail/?cveId=" + v.CveNum cveRef.CveUrl = append(cveRef.CveUrl, cveUrl1) } } @@ -838,7 +838,7 @@ func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, var cveReference0 CveReference cveUrlSlice0 := make([]CveUrl, 0) var cveUrl0 CveUrl - cveUrl0.Url = secLinkConfig + "/en/security/safety-bulletin/detail.html?id=" + localOpenEulerSANum + cveUrl0.Url = secLinkConfig + "/en/security/security-bulletins/detail?id=" + localOpenEulerSANum cveUrlSlice0 = append(cveUrlSlice0, cveUrl0) cveReference0.Type = "Self" cveReference0.CveUrl = cveUrlSlice0 @@ -847,7 +847,7 @@ func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, cveUrlSlice1 := make([]CveUrl, 0) for _, cveNum := range localCveNum { var cveUrl1 CveUrl - cveUrl1.Url = secLinkConfig + "/en/security/cve/detail.html?id=" + cveNum + cveUrl1.Url = secLinkConfig + "/en/security/cve/detail/?cveId=" + cveNum cveUrlSlice1 = append(cveUrlSlice1, cveUrl1) } cveReference1.Type = "openEuler CVE" @@ -1146,7 +1146,7 @@ func BuildVulnerability(vlLenth int, v models.ExcelExport, localOpenEulerSANum = componentInfo.CveNumMap[cpe].OpenEulerSANum } secLinkConfig := beego.AppConfig.String("reflink::openeuler_web") - remediation.Url = secLinkConfig + "/en/security/safety-bulletin/detail.html?id=" + localOpenEulerSANum + remediation.Url = secLinkConfig + "/en/security/security-bulletins/detail/?id=" + localOpenEulerSANum remediations.Remediation = &remediation vulnerability.Remediations = &remediations vulnerabilitySlice = append(vulnerabilitySlice, vulnerability) @@ -1379,7 +1379,7 @@ func PostFile(filename string, targetUrl string) error { return upfileErr } -//downlaod file +// downlaod file func GetFile(url, filePath string) error { res, err := http.Get(url) if err != nil {