From 4c63d8a765eed0c1c657cfbce2d3be20b311be47 Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Mon, 11 Nov 2024 10:06:08 +0800 Subject: [PATCH 1/5] early generate hotpatch updateinfo file --- cve-vulner-manager/Dockerfile | 3 +- .../cve-ddd/app/refactor_hotpatch.go | 43 +++++- .../cve-ddd/domain/hotpatch/hotpatch.go | 2 + .../cve-ddd/domain/updateinfo.go | 24 ++- .../infrastructure/hotpatchimpl/impl.go | 139 ++++++++++++++++-- .../infrastructure/repositoryimpl/impl.go | 10 +- .../updateinfoimpl/generate_updateinfoxml.go | 23 +-- 7 files changed, 209 insertions(+), 35 deletions(-) diff --git a/cve-vulner-manager/Dockerfile b/cve-vulner-manager/Dockerfile index c761407..e4dd787 100644 --- a/cve-vulner-manager/Dockerfile +++ b/cve-vulner-manager/Dockerfile @@ -20,6 +20,7 @@ COPY --chown=manager --from=BUILDER /go/src/gitee.com/openeuler/cve-manager/cve- USER manager WORKDIR /opt/app -RUN mkdir logs +RUN mkdir logs && \ + chmod 550 /opt/app/epoch.sh ENTRYPOINT ["/opt/app/cve-manager"] \ No newline at end of file diff --git a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go index 0ca2a85..bb07724 100644 --- a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go @@ -31,6 +31,7 @@ func NewRefactorHotPatchService( u updateinfo.UpdateInfo, h hotpatch.HotPatch, l *logrus.Entry, + ) *refactorHotPatchService { return &refactorHotPatchService{ repository: r, @@ -66,14 +67,19 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir string) error { return fmt.Errorf("get %s failed: %w", fileIndex, err) } - maxHotPatchId, err := domain.ParseMaxIdFromIndexTxt(indexContent, domain.BulletinTypeHotPatch) + maxHotPatchId, err := h.hotPatch.MaxHotPatchID() if err != nil { return fmt.Errorf("parse max id failed: %w", err) } for _, issue := range issues { - if exist := h.repository.IssueNumExist(issue.HotIssueNum); exist { - h.log.Errorf("issue num %s exists", issue.HotIssueNum) + isPublished, err1 := h.hotPatch.IsPublished(issue.CveNum[0], issue.Component) + if err1 != nil { + h.log.Errorf("check hotpatch is published, occurred error: %s", err1.Error()) + return err + } + + if isPublished { continue } @@ -83,7 +89,7 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir string) error { Component: issue.Component, }) if err1 != nil { - h.log.Errorf("find cve %s, error %s", issue.CveNum, err.Error()) + h.log.Errorf("find cve %s, error %s", issue.CveNum, err1.Error()) return err } if len(cves) == 0 { @@ -120,6 +126,8 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir string) error { uploadFileName = append(uploadFileName, b.PathAppendToIndexFile()) cvesForUpdateInfo = append(cvesForUpdateInfo, b.Cves...) + + h.uploadUpdateInfoFile(&b) } if err = h.repository.SaveIssueNum(issue.HotIssueNum); err != nil { @@ -185,3 +193,30 @@ func (h *refactorHotPatchService) uploadUpdateInfo(cves domain.Cves) error { return nil } + +func (h *refactorHotPatchService) uploadUpdateInfoFile(bulletin *domain.SecurityBulletin) { + for _, branch := range bulletin.AffectedVersion { + filePath := domain.HotPatchUpdateInfoDir + branch + "/updateinfo.xml" + downloadBys, err := h.obs.Download(filePath) + if err != nil { + h.log.Error(err) + continue + } + + data, err := h.updateInfo.UploadUpdateInfoXml(domain.UpdateParam{ + Sb: bulletin, + Branch: branch, + DownloadBys: downloadBys, + }) + + if err != nil { + h.log.Error(err) + continue + } + + if err = h.obs.Upload(filePath, data); err != nil { + h.log.Error(err) + continue + } + } +} diff --git a/cve-vulner-manager/cve-ddd/domain/hotpatch/hotpatch.go b/cve-vulner-manager/cve-ddd/domain/hotpatch/hotpatch.go index 1493d8c..962f399 100644 --- a/cve-vulner-manager/cve-ddd/domain/hotpatch/hotpatch.go +++ b/cve-vulner-manager/cve-ddd/domain/hotpatch/hotpatch.go @@ -4,4 +4,6 @@ import "cvevulner/cve-ddd/domain" type HotPatch interface { GetIssueInfo() ([]domain.HotPatchIssue, error) + MaxHotPatchID() (int, error) + IsPublished(cveId, packageName string) (bool, error) } diff --git a/cve-vulner-manager/cve-ddd/domain/updateinfo.go b/cve-vulner-manager/cve-ddd/domain/updateinfo.go index bf5dfea..3128451 100644 --- a/cve-vulner-manager/cve-ddd/domain/updateinfo.go +++ b/cve-vulner-manager/cve-ddd/domain/updateinfo.go @@ -5,12 +5,26 @@ import ( ) const ( + // UpdateinfoRootDir is the root directory for early update info files UpdateinfoRootDir = "earlyupdateinfo/" - NoticeTypeCVE = "cve" - NoticeTypeBug = "bug" - CveUrlPrefix = "https://nvd.nist.gov/vuln/detail/" - PkgUrl = "https://repo.openeuler.org/%s/update/%s/Packages/%s" - ScriptPath = "/opt/app/epoch.sh" + + // HotPatchUpdateInfoDir is the directory for hotpatch early update info files + HotPatchUpdateInfoDir = "hotpatch-early-updateinfo/" + + // NoticeTypeCVE is the type for CVE notices + NoticeTypeCVE = "cve" + + // NoticeTypeBug is the type for bug notices + NoticeTypeBug = "bug" + + // CveUrlPrefix is the prefix for CVE URLs + CveUrlPrefix = "https://nvd.nist.gov/vuln/detail/" + + // PkgUrl is the URL format for package updates + PkgUrl = "https://repo.openeuler.org/%s/update/%s/Packages/%s" + + // ScriptPath is the path to the epoch.sh script + ScriptPath = "/opt/app/epoch.sh" ) var ( diff --git a/cve-vulner-manager/cve-ddd/infrastructure/hotpatchimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/hotpatchimpl/impl.go index d7d0838..1eb16a7 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/hotpatchimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/hotpatchimpl/impl.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "regexp" + "strconv" "strings" "time" @@ -17,15 +18,22 @@ import ( "cvevulner/util" ) -const hotPatchIssue = "https://gitee.com/api/v5/repos/openeuler/hotpatch_meta/issues?" + - "access_token=%s&state=closed&labels=%s&sort=created&direction=desc&page=%d&per_page=20&created_at=%s" +const ( + hotPatchIssue = "https://gitee.com/api/v5/repos/openeuler/hotpatch_meta/issues?" + + "access_token=%s&state=closed&labels=%s&sort=created&direction=desc&page=%d&per_page=20&created_at=%s" + endpoint = "https://api-cve.openeuler.org" + hotPatchSA = "HotPatchSA" + defaultClientTimeout = 3 + defaultMaxId = 1000 +) var ( - RegexpCve = regexp.MustCompile(`(?s:(.*?))`) - RegexpRPM = regexp.MustCompile(`热补丁路径[:：](?s:(.*?))热补丁信息[:：]`) - RegexpMeta = regexp.MustCompile(`热补丁元数据[:：](?s:(.*?))热补丁路径[:：]`) - RegexpType = regexp.MustCompile(`问题类别[:：](?s:(.*?))热补丁元数据[:：]`) - RegexpInfo = regexp.MustCompile(`热补丁信息[:：](?s:(.*?))$`) + regexpCve = regexp.MustCompile(`(?s:(.*?))`) + regexpRPM = regexp.MustCompile(`热补丁路径[:：](?s:(.*?))热补丁信息[:：]`) + regexpMeta = regexp.MustCompile(`热补丁元数据[:：](?s:(.*?))热补丁路径[:：]`) + regexpType = regexp.MustCompile(`问题类别[:：](?s:(.*?))热补丁元数据[:：]`) + regexpInfo = regexp.MustCompile(`热补丁信息[:：](?s:(.*?))$`) + regepHotPatch = regexp.MustCompile(`openEuler-HotPatchSA-(\d{4})-(\d{4,5})`) ) func NewHotPatchImpl(log *logrus.Entry) *hotPatchImpl { @@ -104,13 +112,13 @@ func (impl hotPatchImpl) getIssues() ([]Issue, error) { } func (impl hotPatchImpl) toPatchIssue(body string) (v domain.HotPatchIssue, err error) { - t := RegexpType.FindAllStringSubmatch(body, -1) + t := regexpType.FindAllStringSubmatch(body, -1) if len(t) == 0 { return v, errors.New("parse type failed") } v.Type = strings.TrimSpace(t[0][1]) - meta := RegexpMeta.FindAllStringSubmatch(body, -1) + meta := regexpMeta.FindAllStringSubmatch(body, -1) if len(meta) == 0 { return v, errors.New("parse metadata failed") } @@ -118,13 +126,13 @@ func (impl hotPatchImpl) toPatchIssue(body string) (v domain.HotPatchIssue, err v.Branch = split[len(split)-4] v.Component = split[len(split)-3] - p := RegexpRPM.FindAllStringSubmatch(body, -1) + p := regexpRPM.FindAllStringSubmatch(body, -1) if len(p) == 0 { return v, errors.New("parse rpm failed") } v.PatchUrl = strings.Split(strings.TrimSpace(p[0][1]), "\n") - info := RegexpInfo.FindAllStringSubmatch(body, -1) + info := regexpInfo.FindAllStringSubmatch(body, -1) if len(info) == 0 { return v, errors.New("parse info failed") } @@ -135,7 +143,7 @@ func (impl hotPatchImpl) toPatchIssue(body string) (v domain.HotPatchIssue, err continue } - if cve := RegexpCve.FindAllStringSubmatch(string(bys), -1); len(cve) > 0 { + if cve := regexpCve.FindAllStringSubmatch(string(bys), -1); len(cve) > 0 { v.CveNum = strings.Split(cve[0][1], ",") break } @@ -147,3 +155,110 @@ func (impl hotPatchImpl) toPatchIssue(body string) (v domain.HotPatchIssue, err return } + +type maxIdResult struct { + Code int `json:"code"` + Result string `json:"result"` + Msg string `json:"msg"` +} + +func (impl hotPatchImpl) MaxHotPatchID() (maxId int, err error) { + cli := utils.NewHttpClient(defaultClientTimeout) + + url := fmt.Sprintf("%s/cve-security-notice-server/securitynotice/getMaxHotPatchId", endpoint) + + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return + } + + r, _, err := cli.Download(request) + if err != nil { + return + } + + var res maxIdResult + if err = json.Unmarshal(r, &res); err != nil { + return + } + + if res.Code != 0 { + err = errors.New(res.Msg) + + return + } + + // init id + if res.Result == "" { + return defaultMaxId, nil + } + + match := regepHotPatch.FindAllStringSubmatch(res.Result, -1) + if len(match) == 0 { + err = errors.New("invalid bulletin id") + + return + } + + // reset id to 1000 at new year + if match[0][1] != strconv.Itoa(time.Now().Year()) { + return defaultMaxId, nil + } + + return strconv.Atoi(match[0][2]) +} + +type cveProductPackageResult struct { + Code int `json:"code"` + Result []cveProductPackage `json:"result"` + Msg string `json:"msg"` +} + +type cveProductPackage struct { + Id int64 `json:"id"` + CveId string `json:"cveId"` + PackageName string `json:"packageName"` + ProductName string `json:"productName"` + Status string `json:"status"` + Reason string `json:"reason"` + SecurityNoticeNo string `json:"securityNoticeNo"` + ReleaseTime string `json:"releaseTime"` + Updateime string `json:"updateTime"` + CreateTime time.Time `json:"createTime"` +} + +func (impl hotPatchImpl) IsPublished(cveId, packageName string) (bool, error) { + cli := utils.NewHttpClient(defaultClientTimeout) + + url := fmt.Sprintf("%s/cve-security-notice-server/cvedatabase/getCVEProductPackageList?cveId=%s&packageName=%s", + endpoint, cveId, packageName) + + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return false, err + } + + r, _, err := cli.Download(request) + if err != nil { + return false, err + } + + var res cveProductPackageResult + if err = json.Unmarshal(r, &res); err != nil { + return false, err + } + + if res.Code != 0 { + err = errors.New(res.Msg) + + return false, err + } + + for _, v := range res.Result { + if strings.Contains(v.SecurityNoticeNo, hotPatchSA) { + return true, nil + } + } + + return false, nil +} diff --git a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go index 995717b..b417942 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go @@ -107,14 +107,20 @@ func (impl repositoryImpl) IssueNumExist(num string) bool { func (impl repositoryImpl) SaveIssueNum(num string) error { now := time.Now().Format(time.RFC3339) + o := orm.NewOrm() hotPatch := models.HotPatch{ IssueNum: num, - CreateTime: now, UpdateTime: now, } - o := orm.NewOrm() + if impl.IssueNumExist(num) { + _, err := o.Update(&hotPatch, "updated_at") + return err + } + + hotPatch.CreateTime = now + _, err := o.Insert(&hotPatch) return err diff --git a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go index 10c5576..23ce7f1 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go @@ -18,6 +18,8 @@ import ( "cvevulner/util" ) +const nokeyOffset = 5 + func (impl updateInfoImpl) UploadUpdateInfoXml(param domain.UpdateParam) (data []byte, err error) { var u Updates @@ -205,18 +207,17 @@ func (impl updateInfoImpl) findEpoch(script, branch, filename, arch string, i in return nil, fmt.Errorf("failed to get epoch, pkgUrl is %s", fmt.Sprintf(domain.PkgUrl, branch, a, filename)) } - if err == nil { - if strings.Contains(string(epoch), "404") || strings.Contains(string(epoch), "502") { - continue - } - if ix := bytes.Index(epoch, []byte("NOKEY")); ix > 0 { - epoch = bytes.TrimSpace(epoch[ix+5:]) - } else { - epoch = bytes.TrimSpace(epoch) - } - - return domain.Num.Find(epoch), nil + if strings.Contains(string(epoch), "404") || strings.Contains(string(epoch), "502") { + continue + } + if ix := bytes.Index(epoch, []byte("NOKEY")); ix > 0 { + epoch = bytes.TrimSpace(epoch[ix+nokeyOffset:]) + } else { + epoch = bytes.TrimSpace(epoch) } + + return domain.Num.Find(epoch), nil + } return nil, nil -- Gitee From 8b6f4515cf054334dcb84271816c1d35edd04df9 Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Tue, 19 Nov 2024 11:47:25 +0800 Subject: [PATCH 2/5] fix review --- cve-vulner-manager/cve-ddd/app/bulletin.go | 32 ++-------- .../cve-ddd/app/refactor_hotpatch.go | 40 ++----------- .../cve-ddd/domain/bulletins.go | 28 ++++++++- .../cve-ddd/domain/repository/cve.go | 2 - .../cve-ddd/domain/service/service.go | 39 ++++++++++++ .../cve-ddd/domain/updateinfo.go | 38 ------------ .../infrastructure/repositoryimpl/impl.go | 35 ----------- .../updateinfoimpl/generate_updateinfoxml.go | 59 ++++++++++--------- 8 files changed, 107 insertions(+), 166 deletions(-) create mode 100644 cve-vulner-manager/cve-ddd/domain/service/service.go diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index 569224f..7f52fc3 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -20,6 +20,7 @@ import ( "cvevulner/cve-ddd/domain/majun" "cvevulner/cve-ddd/domain/obs" "cvevulner/cve-ddd/domain/repository" + "cvevulner/cve-ddd/domain/service" "cvevulner/cve-ddd/domain/testresult" "cvevulner/cve-ddd/domain/updateinfo" ) @@ -29,6 +30,8 @@ const ( fileIndex = "index.txt" fileUpdateFixed = "update_fixed.txt" + + updateinfoRootDir = "earlyupdateinfo/" ) type BulletinService interface { @@ -148,7 +151,7 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin updateFixedFiles = append(updateFixedFiles, v.PathAppendToIndexFile()) - b.uploadUpdateInfoFile(&v) + service.UploadUpdateInfoFile(b.obs, b.updateinfo, b.log, &v, updateinfoRootDir) } b.uploadIndexAndFixed(uploadDir, indexContent, updateFixedFiles) @@ -156,33 +159,6 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin return uploadDir, nil } -func (b *bulletinService) uploadUpdateInfoFile(bulletin *domain.SecurityBulletin) { - for _, branch := range bulletin.AffectedVersion { - filePath := domain.UpdateinfoRootDir + branch + "/updateinfo.xml" - downloadBys, err := b.obs.Download(filePath) - if err != nil { - b.log.Error(err) - continue - } - - data, err := b.updateinfo.UploadUpdateInfoXml(domain.UpdateParam{ - Sb: bulletin, - Branch: branch, - DownloadBys: downloadBys, - }) - - if err != nil { - b.log.Error(err) - continue - } - - if err = b.obs.Upload(filePath, data); err != nil { - b.log.Error(err) - continue - } - } -} - func (b *bulletinService) uploadIndexAndFixed(uploadDir, indexContent string, updateFixedFiles []string) { updateFixedContent := strings.TrimSpace(strings.Join(updateFixedFiles, EOF)) newIndexContent := strings.TrimSpace(indexContent) + EOF + updateFixedContent diff --git a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go index bb07724..6e51918 100644 --- a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go @@ -13,11 +13,14 @@ import ( "cvevulner/cve-ddd/domain/hotpatch" "cvevulner/cve-ddd/domain/obs" "cvevulner/cve-ddd/domain/repository" + "cvevulner/cve-ddd/domain/service" "cvevulner/cve-ddd/domain/updateinfo" ) const ( fileHotPatch = "update_hot_patch.txt" + + hotPatchUpdateInfoDir = "hotpatch-early-updateinfo/" ) type RefactorHotPatchService interface { @@ -112,13 +115,13 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir string) error { xmlData, err2 := h.bulletin.GenerateHotPatch(&b) if err2 != nil { - h.log.Errorf("component: %s, to xml error: %s", b.Component, err.Error()) + h.log.Errorf("component: %s, to xml error: %s", b.Component, err2.Error()) continue } path := uploadDir + b.CvrfFileName() if err2 = h.obs.Upload(path, xmlData); err2 != nil { - h.log.Errorf("component: %s, upload to obs error: %s", b.Component, err.Error()) + h.log.Errorf("component: %s, upload to obs error: %s", b.Component, err2.Error()) continue } @@ -127,11 +130,7 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir string) error { cvesForUpdateInfo = append(cvesForUpdateInfo, b.Cves...) - h.uploadUpdateInfoFile(&b) - } - - if err = h.repository.SaveIssueNum(issue.HotIssueNum); err != nil { - h.log.Errorf("save issue num %s error %s", issue.HotIssueNum, err.Error()) + service.UploadUpdateInfoFile(h.obs, h.updateInfo, h.log, &b, hotPatchUpdateInfoDir) } } @@ -193,30 +192,3 @@ func (h *refactorHotPatchService) uploadUpdateInfo(cves domain.Cves) error { return nil } - -func (h *refactorHotPatchService) uploadUpdateInfoFile(bulletin *domain.SecurityBulletin) { - for _, branch := range bulletin.AffectedVersion { - filePath := domain.HotPatchUpdateInfoDir + branch + "/updateinfo.xml" - downloadBys, err := h.obs.Download(filePath) - if err != nil { - h.log.Error(err) - continue - } - - data, err := h.updateInfo.UploadUpdateInfoXml(domain.UpdateParam{ - Sb: bulletin, - Branch: branch, - DownloadBys: downloadBys, - }) - - if err != nil { - h.log.Error(err) - continue - } - - if err = h.obs.Upload(filePath, data); err != nil { - h.log.Error(err) - continue - } - } -} diff --git a/cve-vulner-manager/cve-ddd/domain/bulletins.go b/cve-vulner-manager/cve-ddd/domain/bulletins.go index 291ffcb..873dc37 100644 --- a/cve-vulner-manager/cve-ddd/domain/bulletins.go +++ b/cve-vulner-manager/cve-ddd/domain/bulletins.go @@ -11,8 +11,14 @@ import ( const ( BulletinTypeSA = "SA" - BulletinTypeBa = "BA" + BulletinTypeBA = "BA" BulletinTypeHotPatch = "HotPatchSA" + + // NoticeTypeCVE is the type for CVE notices + NoticeTypeCVE = "cve" + + // NoticeTypeBug is the type for bug notices + NoticeTypeBug = "bug" ) var ( @@ -38,6 +44,26 @@ type Product struct { IsEpol bool } +func (s *SecurityBulletin) IsCveNotice() bool { + return s.GenNoticeType() == NoticeTypeCVE +} + +func (s *SecurityBulletin) GenNoticeType() string { + if strings.Contains(s.Identification, BulletinTypeBA) { + return NoticeTypeBug + } + + if strings.Contains(s.Identification, BulletinTypeHotPatch) { + return NoticeTypeCVE + } + + if strings.Contains(s.Identification, BulletinTypeSA) { + return NoticeTypeCVE + } + + return "" +} + func (s *SecurityBulletin) SetIdentificationOfColdPatch(id int) { s.Identification = fmt.Sprintf("openEuler-SA-%d-%d", util.Year(), id) } diff --git a/cve-vulner-manager/cve-ddd/domain/repository/cve.go b/cve-vulner-manager/cve-ddd/domain/repository/cve.go index dea7743..fc38100 100644 --- a/cve-vulner-manager/cve-ddd/domain/repository/cve.go +++ b/cve-vulner-manager/cve-ddd/domain/repository/cve.go @@ -10,8 +10,6 @@ type Option struct { type CveRepository interface { FindCves(option Option) (domain.Cves, error) MaxBulletinID() (string, error) - IssueNumExist(num string) bool - SaveIssueNum(num string) error GetAllIssue() (data domain.CollectedDataSlice, err error) SetIgnoreStatus(id int64) error GetAllPackage() (list []string, err error) diff --git a/cve-vulner-manager/cve-ddd/domain/service/service.go b/cve-vulner-manager/cve-ddd/domain/service/service.go new file mode 100644 index 0000000..4683569 --- /dev/null +++ b/cve-vulner-manager/cve-ddd/domain/service/service.go @@ -0,0 +1,39 @@ +package service + +import ( + "cvevulner/cve-ddd/domain" + "cvevulner/cve-ddd/domain/obs" + "cvevulner/cve-ddd/domain/updateinfo" + + "github.com/sirupsen/logrus" +) + +func UploadUpdateInfoFile(obs obs.OBS, updateinfo updateinfo.UpdateInfo, + log *logrus.Entry, bulletin *domain.SecurityBulletin, UpdateInfoDir string, +) { + for _, branch := range bulletin.AffectedVersion { + filePath := UpdateInfoDir + branch + "/updateinfo.xml" + downloadBys, err := obs.Download(filePath) + if err != nil { + log.Error(err) + continue + } + + data, err := updateinfo.UploadUpdateInfoXml(domain.UpdateParam{ + Sb: bulletin, + Branch: branch, + DownloadBys: downloadBys, + }) + + if err != nil { + log.Error(err) + continue + } + + if err = obs.Upload(filePath, data); err != nil { + log.Error(err) + continue + } + } + +} diff --git a/cve-vulner-manager/cve-ddd/domain/updateinfo.go b/cve-vulner-manager/cve-ddd/domain/updateinfo.go index 3128451..218dc9d 100644 --- a/cve-vulner-manager/cve-ddd/domain/updateinfo.go +++ b/cve-vulner-manager/cve-ddd/domain/updateinfo.go @@ -1,43 +1,5 @@ package domain -import ( - "regexp" -) - -const ( - // UpdateinfoRootDir is the root directory for early update info files - UpdateinfoRootDir = "earlyupdateinfo/" - - // HotPatchUpdateInfoDir is the directory for hotpatch early update info files - HotPatchUpdateInfoDir = "hotpatch-early-updateinfo/" - - // NoticeTypeCVE is the type for CVE notices - NoticeTypeCVE = "cve" - - // NoticeTypeBug is the type for bug notices - NoticeTypeBug = "bug" - - // CveUrlPrefix is the prefix for CVE URLs - CveUrlPrefix = "https://nvd.nist.gov/vuln/detail/" - - // PkgUrl is the URL format for package updates - PkgUrl = "https://repo.openeuler.org/%s/update/%s/Packages/%s" - - // ScriptPath is the path to the epoch.sh script - ScriptPath = "/opt/app/epoch.sh" -) - -var ( - Severity = map[string]string{ - "critical": "Critical", - "high": "Important", - "medium": "Moderate", - "low": "Low", - } - - Num = regexp.MustCompile(`\d+`) -) - type UpdateParam struct { Sb *SecurityBulletin Branch string diff --git a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go index b417942..4df85a3 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "strings" - "time" "github.com/astaxie/beego/orm" @@ -92,40 +91,6 @@ func (impl repositoryImpl) MaxBulletinID() (string, error) { return saFileRecord.FileName, nil } -func (impl repositoryImpl) IssueNumExist(num string) bool { - hotPatch := models.HotPatch{ - IssueNum: num, - } - - o := orm.NewOrm() - if err := o.Read(&hotPatch, "issue_num"); err != nil { - return false - } - - return true -} - -func (impl repositoryImpl) SaveIssueNum(num string) error { - now := time.Now().Format(time.RFC3339) - o := orm.NewOrm() - - hotPatch := models.HotPatch{ - IssueNum: num, - UpdateTime: now, - } - - if impl.IssueNumExist(num) { - _, err := o.Update(&hotPatch, "updated_at") - return err - } - - hotPatch.CreateTime = now - - _, err := o.Insert(&hotPatch) - - return err -} - type list struct { models.IssueTemplate AffectProduct string `orm:"column(affect_product)"` diff --git a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go index 23ce7f1..3537b93 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go @@ -5,6 +5,7 @@ import ( "encoding/xml" "fmt" "path/filepath" + "regexp" "sort" "strconv" "strings" @@ -18,7 +19,29 @@ import ( "cvevulner/util" ) -const nokeyOffset = 5 +const ( + nokeyOffset = 5 + + // CveUrlPrefix is the prefix for CVE URLs + CveUrlPrefix = "https://nvd.nist.gov/vuln/detail/" + + // PkgUrl is the URL format for package updates + PkgUrl = "https://repo.openeuler.org/%s/update/%s/Packages/%s" + + // ScriptPath is the path to the epoch.sh script + ScriptPath = "/opt/app/epoch.sh" +) + +var ( + Severity = map[string]string{ + "critical": "Critical", + "high": "Important", + "medium": "Moderate", + "low": "Low", + } + + Num = regexp.MustCompile(`\d+`) +) func (impl updateInfoImpl) UploadUpdateInfoXml(param domain.UpdateParam) (data []byte, err error) { var u Updates @@ -90,7 +113,7 @@ func (impl updateInfoImpl) updateXml(sb *domain.SecurityBulletin, branch string) title := introduction + branch - if impl.IsCveNotice(sb.Identification) { + if sb.IsCveNotice() { if i := strings.Index(description, "Security Fix(es):"); i > 0 { descr = util.TrimStringNR(description[i+17:]) } @@ -104,7 +127,7 @@ func (impl updateInfoImpl) updateXml(sb *domain.SecurityBulletin, branch string) Status: "stable", Id: sb.Identification, Title: title, - Severity: domain.Severity[strings.ToLower(dp.SequenceSeverityLevel[highestLevelIndex])], + Severity: Severity[strings.ToLower(dp.SequenceSeverityLevel[highestLevelIndex])], Release: "openEuler", Issued: &Issued{Date: sb.Date}, Description: descr, @@ -113,7 +136,7 @@ func (impl updateInfoImpl) updateXml(sb *domain.SecurityBulletin, branch string) var ref []Reference for _, s := range cveNums { ref = append(ref, Reference{ - Href: domain.CveUrlPrefix + s, + Href: CveUrlPrefix + s, Id: s, Title: s, Type: "cve", @@ -144,7 +167,7 @@ func (impl updateInfoImpl) updateXml(sb *domain.SecurityBulletin, branch string) } if !strings.Contains(pe.Filename, "kernel") { - epoch, err := impl.findEpoch(domain.ScriptPath, branch, pe.Filename, pe.Arch, 1) + epoch, err := impl.findEpoch(ScriptPath, branch, pe.Filename, pe.Arch, 1) if err == nil && len(epoch) > 0 { pe.Epoch = string(epoch) } @@ -171,26 +194,6 @@ func (impl updateInfoImpl) numberIndex(u *Updates, securityNumber string) (index return } -func (impl updateInfoImpl) IsCveNotice(securityNoticeNo string) bool { - return impl.GenNoticeType(securityNoticeNo) == domain.NoticeTypeCVE -} - -func (impl updateInfoImpl) GenNoticeType(securityNoticeNo string) string { - if strings.Contains(securityNoticeNo, "BA") { - return domain.NoticeTypeBug - } - - if strings.Contains(securityNoticeNo, "HotPatchSA") { - return domain.NoticeTypeCVE - } - - if strings.Contains(securityNoticeNo, "SA") { - return domain.NoticeTypeCVE - } - - return "" -} - func (impl updateInfoImpl) findEpoch(script, branch, filename, arch string, i int) ([]byte, error) { var archs = []string{arch} if arch == "noarch" { @@ -200,11 +203,11 @@ func (impl updateInfoImpl) findEpoch(script, branch, filename, arch string, i in epoch, err, _ := libutils.RunCmd( script, filepath.Join("/opt/app/", branch, strconv.Itoa(i), time.Now().Format("150405.999")), - fmt.Sprintf(domain.PkgUrl, branch, a, filename), + fmt.Sprintf(PkgUrl, branch, a, filename), ) if err != nil { - return nil, fmt.Errorf("failed to get epoch, pkgUrl is %s", fmt.Sprintf(domain.PkgUrl, branch, a, filename)) + return nil, fmt.Errorf("failed to get epoch, pkgUrl is %s", fmt.Sprintf(PkgUrl, branch, a, filename)) } if strings.Contains(string(epoch), "404") || strings.Contains(string(epoch), "502") { @@ -216,7 +219,7 @@ func (impl updateInfoImpl) findEpoch(script, branch, filename, arch string, i in epoch = bytes.TrimSpace(epoch) } - return domain.Num.Find(epoch), nil + return Num.Find(epoch), nil } -- Gitee From 9f2727e025e36bb67195584f6952920f522eac00 Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Tue, 19 Nov 2024 14:25:48 +0800 Subject: [PATCH 3/5] fix codecheck problems --- cve-vulner-manager/cve-ddd/domain/bulletins.go | 2 ++ cve-vulner-manager/cve-ddd/domain/service/service.go | 6 ++++-- .../cve-ddd/infrastructure/hotpatchimpl/impl.go | 3 +-- .../updateinfoimpl/generate_updateinfoxml.go | 8 ++++---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cve-vulner-manager/cve-ddd/domain/bulletins.go b/cve-vulner-manager/cve-ddd/domain/bulletins.go index 873dc37..ddf93e6 100644 --- a/cve-vulner-manager/cve-ddd/domain/bulletins.go +++ b/cve-vulner-manager/cve-ddd/domain/bulletins.go @@ -44,10 +44,12 @@ type Product struct { IsEpol bool } +// IsCveNotice returns whether the security bulletin is a CVE notice. func (s *SecurityBulletin) IsCveNotice() bool { return s.GenNoticeType() == NoticeTypeCVE } +// GenNoticeType returns the type of the security bulletin. func (s *SecurityBulletin) GenNoticeType() string { if strings.Contains(s.Identification, BulletinTypeBA) { return NoticeTypeBug diff --git a/cve-vulner-manager/cve-ddd/domain/service/service.go b/cve-vulner-manager/cve-ddd/domain/service/service.go index 4683569..53a3af5 100644 --- a/cve-vulner-manager/cve-ddd/domain/service/service.go +++ b/cve-vulner-manager/cve-ddd/domain/service/service.go @@ -1,13 +1,15 @@ +// Package service provides the security bulletin service package service import ( + "github.com/sirupsen/logrus" + "cvevulner/cve-ddd/domain" "cvevulner/cve-ddd/domain/obs" "cvevulner/cve-ddd/domain/updateinfo" - - "github.com/sirupsen/logrus" ) +// UploadUpdateInfoFile upload updateinfo file func UploadUpdateInfoFile(obs obs.OBS, updateinfo updateinfo.UpdateInfo, log *logrus.Entry, bulletin *domain.SecurityBulletin, UpdateInfoDir string, ) { diff --git a/cve-vulner-manager/cve-ddd/infrastructure/hotpatchimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/hotpatchimpl/impl.go index 1eb16a7..4689373 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/hotpatchimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/hotpatchimpl/impl.go @@ -22,7 +22,6 @@ const ( hotPatchIssue = "https://gitee.com/api/v5/repos/openeuler/hotpatch_meta/issues?" + "access_token=%s&state=closed&labels=%s&sort=created&direction=desc&page=%d&per_page=20&created_at=%s" endpoint = "https://api-cve.openeuler.org" - hotPatchSA = "HotPatchSA" defaultClientTimeout = 3 defaultMaxId = 1000 ) @@ -255,7 +254,7 @@ func (impl hotPatchImpl) IsPublished(cveId, packageName string) (bool, error) { } for _, v := range res.Result { - if strings.Contains(v.SecurityNoticeNo, hotPatchSA) { + if strings.Contains(v.SecurityNoticeNo, domain.BulletinTypeHotPatch) { return true, nil } } diff --git a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go index 3537b93..0ad4382 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go @@ -33,14 +33,14 @@ const ( ) var ( - Severity = map[string]string{ + severity = map[string]string{ "critical": "Critical", "high": "Important", "medium": "Moderate", "low": "Low", } - Num = regexp.MustCompile(`\d+`) + num = regexp.MustCompile(`\d+`) ) func (impl updateInfoImpl) UploadUpdateInfoXml(param domain.UpdateParam) (data []byte, err error) { @@ -127,7 +127,7 @@ func (impl updateInfoImpl) updateXml(sb *domain.SecurityBulletin, branch string) Status: "stable", Id: sb.Identification, Title: title, - Severity: Severity[strings.ToLower(dp.SequenceSeverityLevel[highestLevelIndex])], + Severity: severity[strings.ToLower(dp.SequenceSeverityLevel[highestLevelIndex])], Release: "openEuler", Issued: &Issued{Date: sb.Date}, Description: descr, @@ -219,7 +219,7 @@ func (impl updateInfoImpl) findEpoch(script, branch, filename, arch string, i in epoch = bytes.TrimSpace(epoch) } - return Num.Find(epoch), nil + return num.Find(epoch), nil } -- Gitee From 8a73ea21147a8cf1a1c380c6b5239d5bea1922e6 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Thu, 21 Nov 2024 16:25:19 +0800 Subject: [PATCH 4/5] fix sql for hot patch --- cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go index 4df85a3..5aeebb3 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go @@ -30,7 +30,6 @@ join cve_issue_template b on a.cve_id=b.cve_id join cve_security_notice c on a.cve_id=c.cve_id where a.cve_num in (%s) and a.cve_status = 2 -and a.is_export in (0,3) and a.organizate_id = 1 and b.status < 4 ` -- Gitee From 443cd02e3afbdbf9b03683dd486d951ec5de1198 Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Thu, 21 Nov 2024 16:32:20 +0800 Subject: [PATCH 5/5] fix pkg url bug --- cve-vulner-manager/cve-ddd/app/bulletin.go | 2 +- .../cve-ddd/app/refactor_hotpatch.go | 6 +++--- cve-vulner-manager/cve-ddd/controller/cve.go | 2 +- .../cve-ddd/domain/service/service.go | 3 ++- cve-vulner-manager/cve-ddd/domain/updateinfo.go | 1 + .../updateinfoimpl/generate_updateinfoxml.go | 14 +++++++------- 6 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cve-vulner-manager/cve-ddd/app/bulletin.go b/cve-vulner-manager/cve-ddd/app/bulletin.go index 7f52fc3..0c5b514 100644 --- a/cve-vulner-manager/cve-ddd/app/bulletin.go +++ b/cve-vulner-manager/cve-ddd/app/bulletin.go @@ -151,7 +151,7 @@ func (b *bulletinService) GenerateBulletins(cveNum []string, date string) (strin updateFixedFiles = append(updateFixedFiles, v.PathAppendToIndexFile()) - service.UploadUpdateInfoFile(b.obs, b.updateinfo, b.log, &v, updateinfoRootDir) + service.UploadUpdateInfoFile(b.obs, b.updateinfo, b.log, &v, date, updateinfoRootDir) } b.uploadIndexAndFixed(uploadDir, indexContent, updateFixedFiles) diff --git a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go index 6e51918..fe52ee1 100644 --- a/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/refactor_hotpatch.go @@ -24,7 +24,7 @@ const ( ) type RefactorHotPatchService interface { - GenerateBulletins(string) error + GenerateBulletins(uploadDir, date string) error } func NewRefactorHotPatchService( @@ -56,7 +56,7 @@ type refactorHotPatchService struct { log *logrus.Entry } -func (h *refactorHotPatchService) GenerateBulletins(uploadDir string) error { +func (h *refactorHotPatchService) GenerateBulletins(uploadDir, date string) error { issues, err := h.hotPatch.GetIssueInfo() if err != nil { return err @@ -130,7 +130,7 @@ func (h *refactorHotPatchService) GenerateBulletins(uploadDir string) error { cvesForUpdateInfo = append(cvesForUpdateInfo, b.Cves...) - service.UploadUpdateInfoFile(h.obs, h.updateInfo, h.log, &b, hotPatchUpdateInfoDir) + service.UploadUpdateInfoFile(h.obs, h.updateInfo, h.log, &b, date, hotPatchUpdateInfoDir) } } diff --git a/cve-vulner-manager/cve-ddd/controller/cve.go b/cve-vulner-manager/cve-ddd/controller/cve.go index 6a19efd..f48a68a 100644 --- a/cve-vulner-manager/cve-ddd/controller/cve.go +++ b/cve-vulner-manager/cve-ddd/controller/cve.go @@ -98,7 +98,7 @@ func (c *CveController) Generate() { if err1 != nil { c.BulletinLog.Errorf("generate security bulletins failed: %s", err1.Error()) } else { - err1 = c.HotPatchService.GenerateBulletins(uploadDir) + err1 = c.HotPatchService.GenerateBulletins(uploadDir, request.Date) if err1 != nil { c.HotPatchBulletinLog.Errorf("generate hot patch security bulletins failed: %s", err1.Error()) } diff --git a/cve-vulner-manager/cve-ddd/domain/service/service.go b/cve-vulner-manager/cve-ddd/domain/service/service.go index 53a3af5..341950c 100644 --- a/cve-vulner-manager/cve-ddd/domain/service/service.go +++ b/cve-vulner-manager/cve-ddd/domain/service/service.go @@ -11,7 +11,7 @@ import ( // UploadUpdateInfoFile upload updateinfo file func UploadUpdateInfoFile(obs obs.OBS, updateinfo updateinfo.UpdateInfo, - log *logrus.Entry, bulletin *domain.SecurityBulletin, UpdateInfoDir string, + log *logrus.Entry, bulletin *domain.SecurityBulletin, date, UpdateInfoDir string, ) { for _, branch := range bulletin.AffectedVersion { filePath := UpdateInfoDir + branch + "/updateinfo.xml" @@ -23,6 +23,7 @@ func UploadUpdateInfoFile(obs obs.OBS, updateinfo updateinfo.UpdateInfo, data, err := updateinfo.UploadUpdateInfoXml(domain.UpdateParam{ Sb: bulletin, + Date: date, Branch: branch, DownloadBys: downloadBys, }) diff --git a/cve-vulner-manager/cve-ddd/domain/updateinfo.go b/cve-vulner-manager/cve-ddd/domain/updateinfo.go index 218dc9d..3f6b880 100644 --- a/cve-vulner-manager/cve-ddd/domain/updateinfo.go +++ b/cve-vulner-manager/cve-ddd/domain/updateinfo.go @@ -2,6 +2,7 @@ package domain type UpdateParam struct { Sb *SecurityBulletin + Date string Branch string DownloadBys []byte } diff --git a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go index 0ad4382..513d44b 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/updateinfoimpl/generate_updateinfoxml.go @@ -26,7 +26,7 @@ const ( CveUrlPrefix = "https://nvd.nist.gov/vuln/detail/" // PkgUrl is the URL format for package updates - PkgUrl = "https://repo.openeuler.org/%s/update/%s/Packages/%s" + PkgUrl = "https://dailybuild.openeuler.openatom.cn/repo.openeuler.org/%s/%s/%s/Packages/%s" // ScriptPath is the path to the epoch.sh script ScriptPath = "/opt/app/epoch.sh" @@ -51,7 +51,7 @@ func (impl updateInfoImpl) UploadUpdateInfoXml(param domain.UpdateParam) (data [ return nil, err } - up := impl.updateXml(param.Sb, param.Branch) + up := impl.updateXml(param.Sb, param.Branch, param.Date) i := impl.numberIndex(&u, param.Sb.Identification) if i == -1 { @@ -81,7 +81,7 @@ func (impl updateInfoImpl) UploadUpdateInfoXml(param domain.UpdateParam) (data [ return headerBytes, nil } -func (impl updateInfoImpl) updateXml(sb *domain.SecurityBulletin, branch string) Update { +func (impl updateInfoImpl) updateXml(sb *domain.SecurityBulletin, branch, date string) Update { var cveNums []string var description string var highestLevelIndex int @@ -167,7 +167,7 @@ func (impl updateInfoImpl) updateXml(sb *domain.SecurityBulletin, branch string) } if !strings.Contains(pe.Filename, "kernel") { - epoch, err := impl.findEpoch(ScriptPath, branch, pe.Filename, pe.Arch, 1) + epoch, err := impl.findEpoch(ScriptPath, branch, pe.Filename, pe.Arch, date, 1) if err == nil && len(epoch) > 0 { pe.Epoch = string(epoch) } @@ -194,7 +194,7 @@ func (impl updateInfoImpl) numberIndex(u *Updates, securityNumber string) (index return } -func (impl updateInfoImpl) findEpoch(script, branch, filename, arch string, i int) ([]byte, error) { +func (impl updateInfoImpl) findEpoch(script, branch, filename, arch, date string, i int) ([]byte, error) { var archs = []string{arch} if arch == "noarch" { archs = []string{"aarch64", "x86_64"} @@ -203,11 +203,11 @@ func (impl updateInfoImpl) findEpoch(script, branch, filename, arch string, i in epoch, err, _ := libutils.RunCmd( script, filepath.Join("/opt/app/", branch, strconv.Itoa(i), time.Now().Format("150405.999")), - fmt.Sprintf(PkgUrl, branch, a, filename), + fmt.Sprintf(PkgUrl, branch, date, a, filename), ) if err != nil { - return nil, fmt.Errorf("failed to get epoch, pkgUrl is %s", fmt.Sprintf(PkgUrl, branch, a, filename)) + return nil, fmt.Errorf("failed to get epoch, pkgUrl is %s", fmt.Sprintf(PkgUrl, date, branch, a, filename)) } if strings.Contains(string(epoch), "404") || strings.Contains(string(epoch), "502") { -- Gitee