From 71279563345b5dc9ce6654b5345683ecdaedddfe Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Mon, 4 Sep 2023 11:50:56 +0800 Subject: [PATCH 1/7] fix ide error --- cve-vulner-manager/tests/createrepo.go | 28 +- cve-vulner-manager/tests/createrepo_test.go | 283 ++++++++++---------- 2 files changed, 156 insertions(+), 155 deletions(-) diff --git a/cve-vulner-manager/tests/createrepo.go b/cve-vulner-manager/tests/createrepo.go index 01ff0d4..f19ef9d 100644 --- a/cve-vulner-manager/tests/createrepo.go +++ b/cve-vulner-manager/tests/createrepo.go @@ -1,21 +1,23 @@ package test import ( - "cvevulner/common" - "cvevulner/models" - "cvevulner/util" "errors" "fmt" + "os" + "github.com/astaxie/beego/logs" "github.com/astaxie/beego/orm" - "os" + + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" ) func QueryOriginCveExcelData() ([]models.OriginExcel, int64, error) { o := orm.NewOrm() var coe []models.OriginExcel num, err := o.Raw("select cve_id,cve_num,cve_url, cve_version, pack_name," + - "score_type,nvd_score,cve_level,cve_desc,repair_time,vector_value,attack_vector,access_vector,attack_complexity,"+ + "score_type,nvd_score,cve_level,cve_desc,repair_time,vector_value,attack_vector,access_vector,attack_complexity," + "access_complexity,privilege_required,user_interaction,scope,confidentiality,integrity,availability,authentication," + "cve_status,create_time from cve_origin_excel GROUP BY pack_name").QueryRows(&coe) if err == nil && num > 0 { @@ -28,7 +30,7 @@ func QueryOriginCveExcelData() ([]models.OriginExcel, int64, error) { func UpdateExcelDataPkName(originPackName, packName string) (id int64) { o := orm.NewOrm() - _ = o.Raw("UPDATE cve_origin_excel SET pack_name = ?" + + _ = o.Raw("UPDATE cve_origin_excel SET pack_name = ?"+ " WHERE pack_name = ?", packName, originPackName).QueryRow() return } @@ -42,7 +44,7 @@ func CreateRepo() { } } -func PostRepo(packName string) error{ +func PostRepo(packName string) error { url := "https://gitee.com/api/v5/user/repos" accessToken := os.Getenv("GITEE_TOKEN") requestBody := fmt.Sprintf(`{ @@ -56,9 +58,9 @@ func PostRepo(packName string) error{ "private": "false" }`, accessToken, packName, packName) logs.Info("isssue_body: ", requestBody) - resp, err := util.HttpPost(url, requestBody) + resp, err := util.HTTPPost(url, requestBody) if err != nil { - logs.Error("创建",packName,"失败, err: ", err) + logs.Error("创建", packName, "失败, err: ", err) return err } if _, ok := resp["id"]; !ok { @@ -70,7 +72,7 @@ func PostRepo(packName string) error{ return nil } -func CreateBrand(packName string) error{ +func CreateBrand(packName string) error { url := "https://gitee.com/api/v5/repos/zhangjianjun_code/" + packName + "/branches" accessToken := os.Getenv("GITEE_TOKEN") requestBody := fmt.Sprintf(`{ @@ -79,9 +81,9 @@ func CreateBrand(packName string) error{ "branch_name": "openEuler-20.03-LTS" }`, accessToken) logs.Info("isssue_body: ", requestBody) - resp, err := util.HttpPost(url, requestBody) + resp, err := util.HTTPPost(url, requestBody) if err != nil { - logs.Error("创建分支：openEuler-20.03-LTS",packName,"失败, err: ", err) + logs.Error("创建分支：openEuler-20.03-LTS", packName, "失败, err: ", err) return err } if _, ok := resp["id"]; !ok { @@ -94,4 +96,4 @@ func CreateBrand(packName string) error{ func Init() { CreateRepo() -} \ No newline at end of file +} diff --git a/cve-vulner-manager/tests/createrepo_test.go b/cve-vulner-manager/tests/createrepo_test.go index 3750e66..52cbc0a 100644 --- a/cve-vulner-manager/tests/createrepo_test.go +++ b/cve-vulner-manager/tests/createrepo_test.go @@ -1,144 +1,143 @@ package test -import ( - "cvevulner/common" - "cvevulner/models" - "cvevulner/util" - "errors" - "fmt" - "github.com/astaxie/beego/logs" - "github.com/astaxie/beego/orm" - "os" -) - -func QueryOriginCveExcelData() ([]models.OriginExcel, int64, error) { - o := orm.NewOrm() - var coe []models.OriginExcel - num, err := o.Raw("select cve_id,cve_num,cve_url, cve_version, pack_name," + - "score_type,nvd_score,cve_level,cve_desc,repair_time,vector_value,attack_vector,access_vector,attack_complexity,"+ - "access_complexity,privilege_required,user_interaction,scope,confidentiality,integrity,availability,authentication," + - "cve_status,create_time from cve_origin_excel GROUP BY pack_name").QueryRows(&coe) - if err == nil && num > 0 { - logs.Info("cve_origin_excel 查询结果： ", num) - } else { - logs.Info("当前无新增或者更新的cve, cur_time:", common.GetCurTime(), "err: ", err) - } - return coe, num, err -} - -func QueryOriginCveCenterData() ([]models.VulnCenter, int64, error) { - o := orm.NewOrm() - var coe []models.VulnCenter - num, err := o.Raw("select * from cve_vuln_center GROUP BY pack_name").QueryRows(&coe) - if err == nil && num > 0 { - logs.Info("cve_vuln_center 查询结果： ", num) - } else { - logs.Info("当前无新增或者更新的cve, cur_time:", common.GetCurTime(), "err: ", err) - } - return coe, num, err -} - -func UpdateExcelDataPkName(originPackName, packName string) (id int64) { - o := orm.NewOrm() - _ = o.Raw("UPDATE cve_origin_excel SET pack_name = ?" + - " WHERE pack_name = ?", packName, originPackName).QueryRow() - return -} - -func CreateRepo() { - //coe, num, err := QueryOriginCveExcelData() - coe, num, err := QueryOriginCveCenterData() - if err == nil && num > 0 { - for _, qc := range coe { - //PostRepo(qc.PackName) - PostGroupRepo(qc.PackName) - //CreateBrand(qc.PackName) - } - } -} - -func PostGroupRepo(packName string) error { - url := "https://gitee.com/api/v5/orgs/cve-test/repos" - accessToken := os.Getenv("GITEE_TOKEN") - requestBody := fmt.Sprintf(`{ - "access_token": "%s", - "name": "%s", - "has_issues": "true", - "has_wiki": "true", - "can_comment": "true", - "auto_init": "true", - "path": "%s", - "private": "false" - }`, accessToken, packName, packName) - logs.Info("isssue_body: ", requestBody) - resp, err := util.HTTPPost(url, requestBody) - if err != nil { - logs.Error("创建",packName,"失败, err: ", err) - return err - } - if _, ok := resp["id"]; !ok { - logs.Error("创建issue 失败, err: ", ok, "url: ", url) - return errors.New("创建仓库失败") - } - errb := CreateBrand(packName) - logs.Info(errb) - return nil -} - -func PostRepo(packName string) error{ - url := "https://gitee.com/api/v5/user/repos" - accessToken := os.Getenv("GITEE_TOKEN") - requestBody := fmt.Sprintf(`{ - "access_token": "%s", - "name": "%s", - "has_issues": "true", - "has_wiki": "true", - "can_comment": "true", - "auto_init": "true", - "path": "%s", - "private": "false" - }`, accessToken, packName, packName) - logs.Info("isssue_body: ", requestBody) - resp, err := util.HTTPPost(url, requestBody) - if err != nil { - logs.Error("创建",packName,"失败, err: ", err) - return err - } - if _, ok := resp["id"]; !ok { - logs.Error("创建issue 失败, err: ", ok, "url: ", url) - return errors.New("创建仓库失败") - } - errb := CreateBrand(packName) - logs.Info(errb) - return nil -} - -func CreateBrand(packName string) error{ - url := "https://gitee.com/api/v5/repos/cve-test/" + packName + "/branches" - accessToken := os.Getenv("GITEE_TOKEN") - requestBody := fmt.Sprintf(`{ - "access_token": "%s", - "refs": "master", - "branch_name": "openEuler-20.03-LTS" - }`, accessToken) - logs.Info("isssue_body: ", requestBody) - resp, err := util.HTTPPost(url, requestBody) - if err != nil { - logs.Error("创建分支：openEuler-20.03-LTS",packName,"失败, err: ", err) - return err - } - if _, ok := resp["id"]; !ok { - logs.Error("创建分支：openEuler-20.03-LTS 失败, err: ", ok, "url: ", url) - return errors.New("创建分支失败") - } - - return nil -} - -<<<<<<< HEAD - -======= -func Init() { - CreateRepo() -} ->>>>>>> dcc533bfb3e46ca7de687eb69f20e183f2353c67 +// +//import ( +// "cvevulner/common" +// "cvevulner/models" +// "cvevulner/util" +// "errors" +// "fmt" +// "github.com/astaxie/beego/logs" +// "github.com/astaxie/beego/orm" +// "os" +//) +// +//func QueryOriginCveExcelData() ([]models.OriginExcel, int64, error) { +// o := orm.NewOrm() +// var coe []models.OriginExcel +// num, err := o.Raw("select cve_id,cve_num,cve_url, cve_version, pack_name," + +// "score_type,nvd_score,cve_level,cve_desc,repair_time,vector_value,attack_vector,access_vector,attack_complexity,"+ +// "access_complexity,privilege_required,user_interaction,scope,confidentiality,integrity,availability,authentication," + +// "cve_status,create_time from cve_origin_excel GROUP BY pack_name").QueryRows(&coe) +// if err == nil && num > 0 { +// logs.Info("cve_origin_excel 查询结果： ", num) +// } else { +// logs.Info("当前无新增或者更新的cve, cur_time:", common.GetCurTime(), "err: ", err) +// } +// return coe, num, err +//} +// +//func QueryOriginCveCenterData() ([]models.VulnCenter, int64, error) { +// o := orm.NewOrm() +// var coe []models.VulnCenter +// num, err := o.Raw("select * from cve_vuln_center GROUP BY pack_name").QueryRows(&coe) +// if err == nil && num > 0 { +// logs.Info("cve_vuln_center 查询结果： ", num) +// } else { +// logs.Info("当前无新增或者更新的cve, cur_time:", common.GetCurTime(), "err: ", err) +// } +// return coe, num, err +//} +// +//func UpdateExcelDataPkName(originPackName, packName string) (id int64) { +// o := orm.NewOrm() +// _ = o.Raw("UPDATE cve_origin_excel SET pack_name = ?" + +// " WHERE pack_name = ?", packName, originPackName).QueryRow() +// return +//} +// +//func CreateRepo() { +// //coe, num, err := QueryOriginCveExcelData() +// coe, num, err := QueryOriginCveCenterData() +// if err == nil && num > 0 { +// for _, qc := range coe { +// //PostRepo(qc.PackName) +// PostGroupRepo(qc.PackName) +// //CreateBrand(qc.PackName) +// } +// } +//} +// +//func PostGroupRepo(packName string) error { +// url := "https://gitee.com/api/v5/orgs/cve-test/repos" +// accessToken := os.Getenv("GITEE_TOKEN") +// requestBody := fmt.Sprintf(`{ +// "access_token": "%s", +// "name": "%s", +// "has_issues": "true", +// "has_wiki": "true", +// "can_comment": "true", +// "auto_init": "true", +// "path": "%s", +// "private": "false" +// }`, accessToken, packName, packName) +// logs.Info("isssue_body: ", requestBody) +// resp, err := util.HTTPPost(url, requestBody) +// if err != nil { +// logs.Error("创建",packName,"失败, err: ", err) +// return err +// } +// if _, ok := resp["id"]; !ok { +// logs.Error("创建issue 失败, err: ", ok, "url: ", url) +// return errors.New("创建仓库失败") +// } +// errb := CreateBrand(packName) +// logs.Info(errb) +// return nil +//} +// +//func PostRepo(packName string) error{ +// url := "https://gitee.com/api/v5/user/repos" +// accessToken := os.Getenv("GITEE_TOKEN") +// requestBody := fmt.Sprintf(`{ +// "access_token": "%s", +// "name": "%s", +// "has_issues": "true", +// "has_wiki": "true", +// "can_comment": "true", +// "auto_init": "true", +// "path": "%s", +// "private": "false" +// }`, accessToken, packName, packName) +// logs.Info("isssue_body: ", requestBody) +// resp, err := util.HTTPPost(url, requestBody) +// if err != nil { +// logs.Error("创建",packName,"失败, err: ", err) +// return err +// } +// if _, ok := resp["id"]; !ok { +// logs.Error("创建issue 失败, err: ", ok, "url: ", url) +// return errors.New("创建仓库失败") +// } +// errb := CreateBrand(packName) +// logs.Info(errb) +// return nil +//} +// +//func CreateBrand(packName string) error{ +// url := "https://gitee.com/api/v5/repos/cve-test/" + packName + "/branches" +// accessToken := os.Getenv("GITEE_TOKEN") +// requestBody := fmt.Sprintf(`{ +// "access_token": "%s", +// "refs": "master", +// "branch_name": "openEuler-20.03-LTS" +// }`, accessToken) +// logs.Info("isssue_body: ", requestBody) +// resp, err := util.HTTPPost(url, requestBody) +// if err != nil { +// logs.Error("创建分支：openEuler-20.03-LTS",packName,"失败, err: ", err) +// return err +// } +// if _, ok := resp["id"]; !ok { +// logs.Error("创建分支：openEuler-20.03-LTS 失败, err: ", ok, "url: ", url) +// return errors.New("创建分支失败") +// } +// +// return nil +//} +// +// +//func Init() { +// CreateRepo() +//} +// -- Gitee From 248ad20e7aabd5e4ee840b13f1296c75ab39e1be Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Mon, 4 Sep 2023 19:31:24 +0800 Subject: [PATCH 2/7] optimize upload logic of hotpatch --- cve-vulner-manager/controllers/file.go | 5 ++ cve-vulner-manager/cve-ddd/app/hotpatch.go | 46 +++++++++++++++++-- cve-vulner-manager/cve-ddd/domain/obs/obs.go | 3 +- .../cve-ddd/infrastructure/obsimpl/impl.go | 35 ++++++++++++-- 4 files changed, 81 insertions(+), 8 deletions(-) diff --git a/cve-vulner-manager/controllers/file.go b/cve-vulner-manager/controllers/file.go index 72dd9a0..18c1df2 100644 --- a/cve-vulner-manager/controllers/file.go +++ b/cve-vulner-manager/controllers/file.go @@ -15,6 +15,7 @@ import ( "cvevulner/common" "cvevulner/cve-ddd/adapter" + "cvevulner/cve-ddd/infrastructure/obsimpl" "cvevulner/models" "cvevulner/taskhandler" "cvevulner/util" @@ -397,6 +398,10 @@ func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, d if dirErr != nil { logs.Error("dirErr: ", dirErr) } + + // set upload dirname of hotpatch + obsimpl.SetDynamicDir(obsDir) + indexFilePath := filepath.Join(dir, "index.txt") indexObjectName := downloadCvrfDir + "index.txt" downObsErr := taskhandler.ObsDownloadFile(indexObjectName, indexFilePath) diff --git a/cve-vulner-manager/cve-ddd/app/hotpatch.go b/cve-vulner-manager/cve-ddd/app/hotpatch.go index 3a1750e..cc0096f 100644 --- a/cve-vulner-manager/cve-ddd/app/hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/hotpatch.go @@ -4,6 +4,7 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/sirupsen/logrus" @@ -15,6 +16,13 @@ import ( "cvevulner/util" ) +const ( + indexFileName = "index.txt" + updateFixedFileName = "update_fixed.txt" +) + +var updateTxt = []string{indexFileName, updateFixedFileName} + type HotPatchService interface { GenerateBulletins([]CmdToGenerateBulletins) error } @@ -38,6 +46,7 @@ type hotPatchService struct { func (h *hotPatchService) GenerateBulletins(cmds []CmdToGenerateBulletins) error { var cvesForUpdateInfo domain.Cves + var uploadFileName []string for _, cmd := range cmds { if exist := h.repository.IssueNumExist(cmd.HotIssueNum); exist { @@ -81,12 +90,14 @@ func (h *hotPatchService) GenerateBulletins(cmds []CmdToGenerateBulletins) error } fileName := fmt.Sprintf("cvrf-%s.xml", b.Identification) - if err := h.obs.UploadBulletin(fileName, xmlData); err != nil { + if err := h.obs.UploadToDynamicDir(fileName, xmlData); err != nil { logrus.Errorf("component: %s, upload to obs error: %s", b.Component, err.Error()) continue } + uploadFileName = append(uploadFileName, fileName) + cvesForUpdateInfo = append(cvesForUpdateInfo, b.Cves...) if err := h.repository.SetMaxBulletinID(b.Identification); err != nil { @@ -99,9 +110,7 @@ func (h *hotPatchService) GenerateBulletins(cmds []CmdToGenerateBulletins) error } } - if len(cvesForUpdateInfo) == 0 { - return nil - } + h.appendHotPatchToIndex(uploadFileName) return h.uploadUpdateInfo(cvesForUpdateInfo) } @@ -134,6 +143,10 @@ func (h *hotPatchService) generateBulletinId() (string, error) { } func (h *hotPatchService) uploadUpdateInfo(cves domain.Cves) error { + if len(cves) == 0 { + return nil + } + for version, v := range cves.GroupByVersion() { bytes, err := h.updateInfo.Generate(v) if err != nil { @@ -151,3 +164,28 @@ func (h *hotPatchService) uploadUpdateInfo(cves domain.Cves) error { return nil } + +func (h *hotPatchService) appendHotPatchToIndex(files []string) { + if len(files) == 0 { + return + } + + var appendContent string + for _, v := range files { + appendContent += fmt.Sprintf("\n%d/%s", time.Now().Year(), v) + } + + for _, v := range updateTxt { + content, err := h.obs.DownloadFromDynamicDir(v) + if err != nil { + logrus.Errorf("download %s error: %s", v, err.Error()) + continue + } + + content = append(content, appendContent...) + + if err = h.obs.UploadToDynamicDir(v, content); err != nil { + logrus.Errorf("upload %s error: %s", v, err.Error()) + } + } +} diff --git a/cve-vulner-manager/cve-ddd/domain/obs/obs.go b/cve-vulner-manager/cve-ddd/domain/obs/obs.go index 6aa0405..4308c71 100644 --- a/cve-vulner-manager/cve-ddd/domain/obs/obs.go +++ b/cve-vulner-manager/cve-ddd/domain/obs/obs.go @@ -1,6 +1,7 @@ package obs type OBS interface { - UploadBulletin(fileName string, data []byte) error + UploadToDynamicDir(fileName string, data []byte) error + DownloadFromDynamicDir(fileName string) ([]byte, error) UploadUpdateInfo(fileName string, data []byte) error } diff --git a/cve-vulner-manager/cve-ddd/infrastructure/obsimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/obsimpl/impl.go index 73c80e5..5e1ab20 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/obsimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/obsimpl/impl.go @@ -3,6 +3,8 @@ package obsimpl import ( "bytes" "fmt" + "io/ioutil" + "strings" "time" "github.com/astaxie/beego" @@ -11,6 +13,12 @@ import ( var instance *obsImpl +var dynamicDir string + +func SetDynamicDir(dir string) { + dynamicDir = dir +} + func Init() error { cfg := Config{ AccessKey: beego.AppConfig.String("obs::access_key_id"), @@ -43,11 +51,10 @@ type obsImpl struct { cli *obs.ObsClient } -func (impl obsImpl) UploadBulletin(fileName string, data []byte) error { +func (impl obsImpl) UploadToDynamicDir(fileName string, data []byte) error { input := &obs.PutObjectInput{} input.Bucket = impl.cfg.Bucket - nowStr := time.Now().Format("2006-01-02") - input.Key = fmt.Sprintf("%s%s-%s/%s", impl.cfg.Directory, nowStr, "hotpatch", fileName) + input.Key = impl.getDynamicDir() + fileName input.Body = bytes.NewReader(data) _, err := impl.cli.PutObject(input) @@ -55,6 +62,19 @@ func (impl obsImpl) UploadBulletin(fileName string, data []byte) error { return err } +func (impl obsImpl) DownloadFromDynamicDir(fileName string) ([]byte, error) { + input := &obs.GetObjectInput{} + input.Bucket = impl.cfg.Bucket + input.Key = impl.getDynamicDir() + fileName + output, err := impl.cli.GetObject(input) + if err != nil { + return nil, err + } + defer output.Body.Close() + + return ioutil.ReadAll(output.Body) +} + func (impl obsImpl) UploadUpdateInfo(fileName string, data []byte) error { input := &obs.PutObjectInput{} input.Bucket = impl.cfg.Bucket @@ -66,3 +86,12 @@ func (impl obsImpl) UploadUpdateInfo(fileName string, data []byte) error { return err } + +func (impl obsImpl) getDynamicDir() string { + todayStr := time.Now().Format("2006-01-02") + if strings.Contains(dynamicDir, todayStr) { + return dynamicDir + } + + return fmt.Sprintf("%s%s-%s/", impl.cfg.UpdateInfoDir, todayStr, "hotpatch") +} -- Gitee From 26b79c155bb279e4c5137a215ffe0cc6e7155366 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Tue, 5 Sep 2023 11:07:00 +0800 Subject: [PATCH 3/7] remote empty file --- cve-vulner-manager/master | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 cve-vulner-manager/master diff --git a/cve-vulner-manager/master b/cve-vulner-manager/master deleted file mode 100644 index e69de29..0000000 -- Gitee From 3412c0620b7617bcf0af3c85aa8071363c56ab7e Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Tue, 5 Sep 2023 20:26:08 +0800 Subject: [PATCH 4/7] fix bug of description --- cve-vulner-manager/models/giteeissue.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cve-vulner-manager/models/giteeissue.go b/cve-vulner-manager/models/giteeissue.go index a86404d..0742fc6 100644 --- a/cve-vulner-manager/models/giteeissue.go +++ b/cve-vulner-manager/models/giteeissue.go @@ -1,12 +1,14 @@ package models import ( - "cvevulner/common" - "cvevulner/util" "errors" + "strings" + "github.com/astaxie/beego/logs" "github.com/astaxie/beego/orm" - "strings" + + "cvevulner/common" + "cvevulner/util" ) //Loophole issue body model @@ -230,7 +232,7 @@ func parseOldTplToLoopHole(lp *Loophole, body string) { if len(sm) > 0 && len(sm[0]) > 1 { for _, v := range sm[0][1:] { if v != "" { - lp.Version = util.TrimString(v) + lp.Version = strings.Split(util.TrimString(v), ",")[0] break } } @@ -328,7 +330,7 @@ func parseNewTplToLoopHole(lp *Loophole, body string) { if len(sm) > 0 && len(sm[0]) > 1 { for _, v := range sm[0][1:] { if v != "" { - lp.Version = util.TrimString(v) + lp.Version = strings.Split(util.TrimString(v), ",")[0] break } } -- Gitee From 417c52809de4688636b6fe938382a3059487cf73 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Wed, 6 Sep 2023 10:55:43 +0800 Subject: [PATCH 5/7] sa number --- cve-vulner-manager/controllers/file.go | 7 +++- cve-vulner-manager/cve-ddd/app/hotpatch.go | 34 ++++++--------- .../cve-ddd/domain/repository/cve.go | 1 - .../infrastructure/repositoryimpl/impl.go | 41 ++++--------------- cve-vulner-manager/models/excel.go | 9 +++- cve-vulner-manager/models/modeldb.go | 3 +- 6 files changed, 33 insertions(+), 62 deletions(-) diff --git a/cve-vulner-manager/controllers/file.go b/cve-vulner-manager/controllers/file.go index 18c1df2..e4dc70d 100644 --- a/cve-vulner-manager/controllers/file.go +++ b/cve-vulner-manager/controllers/file.go @@ -413,7 +413,7 @@ func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, d SaveFileToDb(saFileStr) totalSlice := make([]string, 0) saNumber := int64(1000) - sfl := models.GetCvrfAllFile() + sfl := models.GetCvrfAllFile(models.SaFileRecordSa) curYears := strconv.Itoa(time.Now().Year()) var oldyear string saDir := beego.AppConfig.DefaultString("saFileDir", "download/sa") @@ -587,6 +587,11 @@ func SaveFileRecord(fileName string) { af.Status = 1 af.FileName = fileName af.CreateTime = common.GetCurTime() + + if strings.Contains(fileName, "HotPatchSA") { + af.SaType = models.SaFileRecordHotPatch + } + models.InsertCvrfFileRecord(&af) } diff --git a/cve-vulner-manager/cve-ddd/app/hotpatch.go b/cve-vulner-manager/cve-ddd/app/hotpatch.go index cc0096f..5e3ca2f 100644 --- a/cve-vulner-manager/cve-ddd/app/hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/hotpatch.go @@ -47,6 +47,10 @@ type hotPatchService struct { func (h *hotPatchService) GenerateBulletins(cmds []CmdToGenerateBulletins) error { var cvesForUpdateInfo domain.Cves var uploadFileName []string + id, err := h.generateBulletinId() + if err != nil { + return err + } for _, cmd := range cmds { if exist := h.repository.IssueNumExist(cmd.HotIssueNum); exist { @@ -76,11 +80,8 @@ func (h *hotPatchService) GenerateBulletins(cmds []CmdToGenerateBulletins) error for _, b := range bulletins { b.PatchUrl = cmd.PatchUrl - id, err := h.generateBulletinId() - if err != nil { - return err - } - b.Identification = id + id++ + b.Identification = fmt.Sprintf("openEuler-HotPatchSA-%d-%d", util.Year(), id) xmlData, err := h.bulletin.Generate(&b) if err != nil { @@ -99,10 +100,6 @@ func (h *hotPatchService) GenerateBulletins(cmds []CmdToGenerateBulletins) error uploadFileName = append(uploadFileName, fileName) cvesForUpdateInfo = append(cvesForUpdateInfo, b.Cves...) - - if err := h.repository.SetMaxBulletinID(b.Identification); err != nil { - logrus.Errorf("set max bulletin id %s error %s", b.Identification, err.Error()) - } } if err := h.repository.SaveIssueNum(cmd.HotIssueNum); err != nil { @@ -115,31 +112,24 @@ func (h *hotPatchService) GenerateBulletins(cmds []CmdToGenerateBulletins) error return h.uploadUpdateInfo(cvesForUpdateInfo) } -func (h *hotPatchService) generateBulletinId() (string, error) { - bulletinNumFormat := "openEuler-HotPatchSA-%d-%d" - +func (h *hotPatchService) generateBulletinId() (int, error) { maxID, err := h.repository.MaxBulletinID() if err != nil { - return "", err + return 0, err } thisYear := util.Year() if maxID == "" { - return fmt.Sprintf(bulletinNumFormat, thisYear, 1001), nil + return 1001, nil } split := strings.Split(maxID, "-") - if split[2] != strconv.Itoa(thisYear) { - return fmt.Sprintf(bulletinNumFormat, thisYear, 1001), nil - } - - num, err := strconv.Atoi(split[3]) - if err != nil { - return "", err + if split[3] != strconv.Itoa(thisYear) { + return 1001, nil } - return fmt.Sprintf(bulletinNumFormat, thisYear, num+1), nil + return strconv.Atoi(split[4]) } func (h *hotPatchService) uploadUpdateInfo(cves domain.Cves) error { diff --git a/cve-vulner-manager/cve-ddd/domain/repository/cve.go b/cve-vulner-manager/cve-ddd/domain/repository/cve.go index b84d3f5..70d61b7 100644 --- a/cve-vulner-manager/cve-ddd/domain/repository/cve.go +++ b/cve-vulner-manager/cve-ddd/domain/repository/cve.go @@ -11,7 +11,6 @@ type Option struct { type CveRepository interface { FindCves(option Option) (domain.Cves, error) MaxBulletinID() (string, error) - SetMaxBulletinID(string) error IssueNumExist(num string) bool SaveIssueNum(num string) error } diff --git a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go index b3072f2..dbadd33 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go @@ -64,10 +64,13 @@ where a.cve_num in (%s) and a.organizate_id = 1 and a.pack_name = "%s" } func (impl repositoryImpl) MaxBulletinID() (string, error) { - var hotPatch models.HotPatch + var saFileRecord models.SaFileRecord o := orm.NewOrm() - err := o.QueryTable(&hotPatch).Filter("type", 2).One(&hotPatch) + err := o.QueryTable(&saFileRecord). + Filter("sa_type", models.SaFileRecordHotPatch). + OrderBy("-sa_number"). + One(&saFileRecord) if errors.Is(err, orm.ErrNoRows) { return "", nil } @@ -76,45 +79,16 @@ func (impl repositoryImpl) MaxBulletinID() (string, error) { return "", err } - return hotPatch.MaxID, nil -} - -func (impl repositoryImpl) SetMaxBulletinID(id string) error { - hotPatch := models.HotPatch{ - Type: 2, - } - now := time.Now().Format(time.RFC3339) - - o := orm.NewOrm() - err := o.Read(&hotPatch, "type") - if errors.Is(err, orm.ErrNoRows) { - hotPatch.MaxID = id - hotPatch.CreateTime = now - hotPatch.UpdateTime = now - - _, err := o.Insert(&hotPatch) - - return err - - } else if err == nil { - hotPatch.MaxID = id - hotPatch.UpdateTime = now - _, err := o.Update(&hotPatch) - - return err - } - - return err + return saFileRecord.FileName, nil } func (impl repositoryImpl) IssueNumExist(num string) bool { hotPatch := models.HotPatch{ - Type: 1, IssueNum: num, } o := orm.NewOrm() - if err := o.Read(&hotPatch, "type", "issue_num"); err != nil { + if err := o.Read(&hotPatch, "issue_num"); err != nil { return false } @@ -125,7 +99,6 @@ func (impl repositoryImpl) SaveIssueNum(num string) error { now := time.Now().Format(time.RFC3339) hotPatch := models.HotPatch{ - Type: 1, IssueNum: num, CreateTime: now, UpdateTime: now, diff --git a/cve-vulner-manager/models/excel.go b/cve-vulner-manager/models/excel.go index a6ebe97..f931ea5 100644 --- a/cve-vulner-manager/models/excel.go +++ b/cve-vulner-manager/models/excel.go @@ -7,6 +7,11 @@ import ( "github.com/astaxie/beego/orm" ) +const ( + SaFileRecordSa = "sa" + SaFileRecordHotPatch = "hotPatch" +) + //ExcelExport the export excel row content model type ExcelExport struct { Num int64 @@ -102,10 +107,10 @@ func GetCvrfFileName(afl *SaFileList, colName ...string) error { return err } -func GetCvrfAllFile() []SaFileRecord { +func GetCvrfAllFile(t string) []SaFileRecord { o := orm.NewOrm() var afl []SaFileRecord - num, err := o.Raw("SELECT * FROM cve_sa_file_record order by file_id asc").QueryRows(&afl) + num, err := o.Raw("SELECT * FROM cve_sa_file_record where sa_type = ? order by file_id asc", t).QueryRows(&afl) if err == nil { logs.Info("cve_sa_file_record nums: ", num) } else { diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index 48b73db..db4dde3 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -763,6 +763,7 @@ type IssueStatisticsMailList struct { type SaFileRecord struct { FileId int64 `orm:"pk;auto;column(file_id)"` + SaType string `orm:"size(255);column(sa_type)" description:"sa的类型"` FileName string `orm:"size(512);column(file_name)" description:"sa的cvrf文件名称"` Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` SaNumber int64 `orm:"column(sa_number)" description:"当前数字"` @@ -1055,8 +1056,6 @@ type IssueDeleteRecord struct { type HotPatch struct { Id int64 `orm:"pk;auto;column(id)"` - Type int64 `orm:"column(type)"` - MaxID string `orm:"size(256);column(max_id);"` IssueNum string `orm:"size(256);column(issue_num);index"` CreateTime string `orm:"size(32);column(created_at)"` UpdateTime string `orm:"size(32);column(updated_at);null"` -- Gitee From dcecfec9e6c36c0afb3ccfe6875d2b5f0fe09338 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Wed, 6 Sep 2023 15:51:29 +0800 Subject: [PATCH 6/7] some optimize --- cve-vulner-manager/controllers/file.go | 5 +-- cve-vulner-manager/cve-ddd/app/hotpatch.go | 44 +++++++++++++--------- cve-vulner-manager/models/excel.go | 5 --- cve-vulner-manager/models/function.go | 16 ++++++++ 4 files changed, 44 insertions(+), 26 deletions(-) create mode 100644 cve-vulner-manager/models/function.go diff --git a/cve-vulner-manager/controllers/file.go b/cve-vulner-manager/controllers/file.go index e4dc70d..5fdf90b 100644 --- a/cve-vulner-manager/controllers/file.go +++ b/cve-vulner-manager/controllers/file.go @@ -587,10 +587,7 @@ func SaveFileRecord(fileName string) { af.Status = 1 af.FileName = fileName af.CreateTime = common.GetCurTime() - - if strings.Contains(fileName, "HotPatchSA") { - af.SaType = models.SaFileRecordHotPatch - } + af.SaType = af.GenSaType() models.InsertCvrfFileRecord(&af) } diff --git a/cve-vulner-manager/cve-ddd/app/hotpatch.go b/cve-vulner-manager/cve-ddd/app/hotpatch.go index 5e3ca2f..7069a28 100644 --- a/cve-vulner-manager/cve-ddd/app/hotpatch.go +++ b/cve-vulner-manager/cve-ddd/app/hotpatch.go @@ -13,6 +13,7 @@ import ( "cvevulner/cve-ddd/domain/obs" "cvevulner/cve-ddd/domain/repository" "cvevulner/cve-ddd/domain/updateinfo" + "cvevulner/models" "cvevulner/util" ) @@ -21,8 +22,6 @@ const ( updateFixedFileName = "update_fixed.txt" ) -var updateTxt = []string{indexFileName, updateFixedFileName} - type HotPatchService interface { GenerateBulletins([]CmdToGenerateBulletins) error } @@ -107,7 +106,7 @@ func (h *hotPatchService) GenerateBulletins(cmds []CmdToGenerateBulletins) error } } - h.appendHotPatchToIndex(uploadFileName) + h.appendHotPatchToFiles(uploadFileName) return h.uploadUpdateInfo(cvesForUpdateInfo) } @@ -121,12 +120,12 @@ func (h *hotPatchService) generateBulletinId() (int, error) { thisYear := util.Year() if maxID == "" { - return 1001, nil + return 1000, nil } - split := strings.Split(maxID, "-") + split := strings.Split(strings.Trim(maxID, ".xml"), "-") if split[3] != strconv.Itoa(thisYear) { - return 1001, nil + return 1000, nil } return strconv.Atoi(split[4]) @@ -155,7 +154,7 @@ func (h *hotPatchService) uploadUpdateInfo(cves domain.Cves) error { return nil } -func (h *hotPatchService) appendHotPatchToIndex(files []string) { +func (h *hotPatchService) appendHotPatchToFiles(files []string) { if len(files) == 0 { return } @@ -165,17 +164,28 @@ func (h *hotPatchService) appendHotPatchToIndex(files []string) { appendContent += fmt.Sprintf("\n%d/%s", time.Now().Year(), v) } - for _, v := range updateTxt { - content, err := h.obs.DownloadFromDynamicDir(v) - if err != nil { - logrus.Errorf("download %s error: %s", v, err.Error()) - continue - } + h.updateFileByContent(updateFixedFileName, appendContent) - content = append(content, appendContent...) + var oldContent string + oldRecords := models.GetCvrfAllFile(models.SaFileRecordHotPatch) + for _, v := range oldRecords { + oldContent += fmt.Sprintf("\n%d/%s", time.Now().Year(), v.FileName) + } - if err = h.obs.UploadToDynamicDir(v, content); err != nil { - logrus.Errorf("upload %s error: %s", v, err.Error()) - } + h.updateFileByContent(indexFileName, oldContent+appendContent) +} + +func (h *hotPatchService) updateFileByContent(file, newContent string) { + oldContent, err := h.obs.DownloadFromDynamicDir(file) + if err != nil { + logrus.Errorf("download %s error: %s", file, err.Error()) + + return + } + + oldContent = append(oldContent, newContent...) + + if err = h.obs.UploadToDynamicDir(file, oldContent); err != nil { + logrus.Errorf("upload %s error: %s", file, err.Error()) } } diff --git a/cve-vulner-manager/models/excel.go b/cve-vulner-manager/models/excel.go index f931ea5..43d76ec 100644 --- a/cve-vulner-manager/models/excel.go +++ b/cve-vulner-manager/models/excel.go @@ -7,11 +7,6 @@ import ( "github.com/astaxie/beego/orm" ) -const ( - SaFileRecordSa = "sa" - SaFileRecordHotPatch = "hotPatch" -) - //ExcelExport the export excel row content model type ExcelExport struct { Num int64 diff --git a/cve-vulner-manager/models/function.go b/cve-vulner-manager/models/function.go new file mode 100644 index 0000000..640289e --- /dev/null +++ b/cve-vulner-manager/models/function.go @@ -0,0 +1,16 @@ +package models + +import "strings" + +const ( + SaFileRecordSa = "sa" + SaFileRecordHotPatch = "hotPatch" +) + +func (sa *SaFileRecord) GenSaType() string { + if strings.Contains(sa.FileName, "HotPatchSA") { + return SaFileRecordHotPatch + } + + return SaFileRecordSa +} -- Gitee From 5287316a9960f20b104901e5681bfa5d6ed767d3 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Wed, 6 Sep 2023 16:07:57 +0800 Subject: [PATCH 7/7] order by filename --- .../cve-ddd/infrastructure/repositoryimpl/impl.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go index dbadd33..ce6b5e3 100644 --- a/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go +++ b/cve-vulner-manager/cve-ddd/infrastructure/repositoryimpl/impl.go @@ -69,7 +69,7 @@ func (impl repositoryImpl) MaxBulletinID() (string, error) { o := orm.NewOrm() err := o.QueryTable(&saFileRecord). Filter("sa_type", models.SaFileRecordHotPatch). - OrderBy("-sa_number"). + OrderBy("-file_name"). One(&saFileRecord) if errors.Is(err, orm.ErrNoRows) { return "", nil -- Gitee