From c5a6a35543c8a30b70ac998bfd3de2e6d277b2a5 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Tue, 1 Apr 2025 15:49:20 +0800 Subject: [PATCH 1/4] clear old package logic --- .../controller/task_controller.go | 21 -- .../controller/timer_task_controller.go | 22 -- .../cve-timed-task/tabletask/common.go | 68 ---- .../cve-timed-task/tabletask/gauss_yaml.go | 308 --------------- .../tabletask/mindspore_yaml.go | 352 ------------------ .../tabletask/parselookeng_yaml.go | 287 -------------- 6 files changed, 1058 deletions(-) delete mode 100644 cve-vulner-manager/cve-timed-task/tabletask/common.go delete mode 100644 cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go delete mode 100644 cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go delete mode 100644 cve-vulner-manager/cve-timed-task/tabletask/parselookeng_yaml.go diff --git a/cve-vulner-manager/cve-timed-task/controller/task_controller.go b/cve-vulner-manager/cve-timed-task/controller/task_controller.go index 9d7345a..60fcba0 100644 --- a/cve-vulner-manager/cve-timed-task/controller/task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/task_controller.go @@ -109,24 +109,3 @@ func LongSupplementCveTask() { tabletask.LongSupplementCve() logs.Info("END TASK:", "long Supplement cve information task is over") } - -// ParseOpenGaussYamlTask Download the yaml file first, then parse the yaml file of opengauss -func ParseOpenGaussYamlTask() { - logs.Info("START TASK:", "the task of Process openGauss yaml data start") - tabletask.ProcGaussYaml() - logs.Info("END TASK:", "the task of Process openGauss yaml data complete") -} - -// ParseMindsporeYamlTask Download the yaml file first, then parse the yaml file of mindspore -func ParseMindsporeYamlTask() { - logs.Info("START TASK:", "the task of Process mindspore yaml data start") - tabletask.ProcMindSporeYaml() - logs.Info("END TASK:", "the task of Process mindspore yaml data complete") -} - -// ParseOpenlookengYamlTask Download the yaml file first, then parse the yaml file of openLooKeng -func ParseOpenlookengYamlTask() { - logs.Info("START TASK:", "the task of Process openLooKeng yaml data start") - tabletask.ProcOpenlookengYaml() - logs.Info("END TASK:", "the task of Process openLooKeng yaml data complete") -} diff --git a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go index 96efc9d..c620bbb 100644 --- a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go @@ -85,29 +85,7 @@ func StartTimerTask() error { return lErr } logs.Info("Add LongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) - //Parse opengauss yaml file - IdParseOpenGaussYamlTask, pErr := c.AddFunc(beego.AppConfig.String("timedtask::ParseOpenGaussYamlTask"), ParseOpenGaussYamlTask) - if pErr != nil { - logs.Error("add ParseOpenGaussYamlTask task error:", pErr.Error()) - return pErr - } - logs.Info("Add ParseOpenGaussYamlTask task success, ID is:", IdParseOpenGaussYamlTask) - - //Parse the yaml file of mindspore - IdParseMindsporeYamlTask, yErr := c.AddFunc(beego.AppConfig.String("timedtask::ParseMindsporeYamlTask"), ParseMindsporeYamlTask) - if yErr != nil { - logs.Error("add ParseMindsporeYamlTask task error:", yErr.Error()) - return yErr - } - logs.Info("Add ParseMindsporeYamlTask task success, ID is:", IdParseMindsporeYamlTask) - //Parse the yaml file of mindspore - IdParseOpenlookengYamlTask, oErr := c.AddFunc(beego.AppConfig.String("timedtask::ParseOpenlookengYamlTask"), ParseOpenlookengYamlTask) - if oErr != nil { - logs.Error("add ParseMindsporeYamlTask task error:", oErr.Error()) - return oErr - } - logs.Info("Add ParseMindsporeYamlTask task success, ID is:", IdParseOpenlookengYamlTask) c.Start() return nil } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/common.go b/cve-vulner-manager/cve-timed-task/tabletask/common.go deleted file mode 100644 index da31523..0000000 --- a/cve-vulner-manager/cve-timed-task/tabletask/common.go +++ /dev/null @@ -1,68 +0,0 @@ -package tabletask - -import ( - "errors" - - "github.com/astaxie/beego/logs" - "github.com/astaxie/beego/orm" - - "cvevulner/cve-timed-task/db_models" -) - -// DeleteYamlData delete yaml data -func DeleteYamlData(ormModel orm.Ormer) { - packagesData, rowsAffected, err := db_models.SelectOpenEulerYamlData(ormModel) - if err != nil { - logs.Error("db_models.SelectOpenEulerYamlData error:", err.Error()) - return - } - if rowsAffected > 0 { - for _, v := range packagesData { - if v.PackageId >= 10000000 { - _, errg := db_models.SelectGaussYamlOriginData(v.PackageName, v.Version, ormModel) - _, errm := db_models.SelectMindSporeYamlOriginData(v.PackageName, v.Version, ormModel) - _, erro := db_models.SelectOpenLookengYamlOriginData(v.PackageName, v.Version, ormModel) - if errors.Is(orm.ErrNoRows, errg) && errors.Is(orm.ErrNoRows, errm) && errors.Is(orm.ErrNoRows, erro) { - logs.Info("ID of the currently deleted data: ", v.GitId) - //open transaction - tranErr := ormModel.Begin() - if tranErr != nil { - logs.Error(" Open transaction error:", tranErr.Error()) - continue - } - err = db_models.DeleteYamlOpenEulerDetailData(v.GitId, ormModel) - if err != nil { - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - err = db_models.DeleteYamlOpenEulerData(v.GitId, ormModel) - if err != nil { - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - tranErr = ormModel.Commit() - if tranErr != nil { - logs.Error(" Commit transaction error:", tranErr.Error()) - continue - } - } else { - if errg != nil && errors.Is(orm.ErrNoRows, errg) { - logs.Error("db_models.SelectGaussYamlOriginData: ", errg, v.PackageName, v.Version) - } - if errm != nil && errors.Is(orm.ErrNoRows, errm) { - logs.Error("db_models.SelectMindSporeYamlOriginData: ", errm, v.PackageName, v.Version) - } - if erro != nil && errors.Is(orm.ErrNoRows, erro) { - logs.Error("db_models.SelectOpenLookengYamlOriginData: ", erro, v.PackageName, v.Version) - } - } - } - } - } -} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go deleted file mode 100644 index 381f25c..0000000 --- a/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go +++ /dev/null @@ -1,308 +0,0 @@ -package tabletask - -import ( - "errors" - "io" - "net/http" - "os" - "strings" - "time" - - "cvevulner/cve-timed-task/db_models" - "cvevulner/cve-timed-task/util" - "cvevulner/models" - - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" - "github.com/astaxie/beego/orm" -) - -// DownloadGuessYaml Download the yaml file of openGauss on Gitee -func DownloadGuessYaml(downloadUrl string) (string, error) { - filePath := beego.AppConfig.String("timedtask::packageDir") + "gauss_yaml/" - fErr := util.MakeDir(filePath) - if fErr != nil { - return "", fErr - } - yamlFileName := filePath + "gauss_yaml.yaml" - //download the yaml file - resp, gErr := http.Get(downloadUrl) - defer resp.Body.Close() - if gErr != nil { - logs.Error("get", downloadUrl, "error: ", gErr.Error()) - return "", gErr - } - if resp.StatusCode != http.StatusOK { - logs.Error("get", downloadUrl, resp.Status) - return "", errors.New(resp.Status) - } - _ = os.Remove(yamlFileName) - saveFile, cErr := os.Create(yamlFileName) - defer saveFile.Close() - if cErr != nil { - logs.Error("Create file error ", yamlFileName, cErr.Error()) - return "", cErr - } - _, yErr := io.Copy(saveFile, resp.Body) - if yErr != nil { - logs.Error("copy resp.Body to file error", yamlFileName, yErr.Error()) - return "", yErr - } - return yamlFileName, nil -} - -func StoreYamlData(yamlData map[string]map[string]string, ormModel orm.Ormer, warehouse string) { - if len(yamlData) > 0 { - for k, v := range yamlData { - var err error - if _, ok := v["version"]; !ok || v["version"] == "" { - v["version"] = "" - } - //open transaction - logs.Info("open transaction") - tranErr := ormModel.Begin() - if tranErr != nil { - logs.Error(" Open transaction error:", tranErr.Error()) - continue - } - originData, sErr := db_models.SelectYamlOriginData(k, v["version"], ormModel) - now := time.Now() - if sErr != nil { - if errors.Is(orm.ErrNoRows, sErr) { - logs.Warn("error is orm.ErrNoRows, Create a record") - openGuessYaml := &models.OpenGussYaml{ - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - Status: 1, - CpeName: v["cpeName"], - CreateTime: now.Format("2006-01-02 15:04:05"), - UpdateTime: now.Format("2006-01-02 15:04:05"), - MarkBit: 1, - Repo: "security", - Warehouse: warehouse, - } - sErr = db_models.InsertYamlOriginData(openGuessYaml, ormModel) - if sErr != nil { - logs.Error("db_models.InsertYamlOriginData error:", sErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - logs.Info("insert OpenGussYaml success") - } else { - logs.Error("db_models.SelectYamlOriginData error:", err) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } else { - logs.Info("database records, change the data") - openGuessYaml := &models.OpenGussYaml{ - Id: originData.Id, - OriginUrl: v["url"], - Status: 1, - CpeName: v["cpeName"], - UpdateTime: now.Format("2006-01-02 15:04:05"), - Repo: "security", - MarkBit: 1, - Warehouse: warehouse, - } - _, err = db_models.UpdateYamlOriginData(openGuessYaml, ormModel) - if err != nil { - logs.Error("db_models.UpdateYamlOriginData:", err.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - packAgesData, rowsAffected, lErr := db_models.SelectYamlData(k, v["version"], ormModel) - if lErr != nil { - logs.Error("db_models.SelectYamlData error:", lErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - if rowsAffected == 0 { - logs.Warn("No record found, Create a record") - var packId int64 - gitOpenEuler, yErr := db_models.SelectYamlLastData(ormModel) - if yErr != nil { - if errors.Is(orm.ErrNoRows, yErr) { - packId = 10000000 - } else { - logs.Error("db_models.SelectYamlLastData error:", yErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } else { - if gitOpenEuler.PackageId >= 10000000 { - packId = gitOpenEuler.PackageId + 1 - } else { - packId = 10000000 - } - } - goe := &models.GitOpenEuler{ - PackageId: packId, - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - CreateTime: now, - UpdateTime: now, - CpePackName: v["cpeName"], - Status: 1, - } - lastId, iErr := db_models.InsertYamlData(goe, ormModel) - if iErr != nil { - logs.Error("db_models.InsertYamlData error:", iErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - gpi := &models.GitPackageInfo{ - GitId: lastId, - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - CreateTime: now, - UpdateTime: now, - Decription: "", - Status: 0, - } - err = db_models.InsertYamlDetailData(gpi, ormModel) - if err != nil { - logs.Error("db_models.InsertYamlDetailData:", err.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } else { - if rowsAffected > 1 { - for _, pv := range packAgesData[1:] { - err = db_models.DeleteYamlOpenEulerDetailData(pv.GitId, ormModel) - if err != nil { - logs.Error("db_models.DeleteYamlOpenEulerDetailData:", err.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - err = db_models.DeleteYamlOpenEulerData(pv.GitId, ormModel) - if err != nil { - logs.Error("db_models.DeleteYamlOpenEulerData:", err.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - } - goe := &models.GitOpenEuler{ - OriginUrl: v["url"], - UpdateTime: now, - CpePackName: v["cpeName"], - Status: 1, - PackageId: packAgesData[0].PackageId, - PackageName: k, - Version: v["version"], - } - err = db_models.UpdateYamlData(goe, ormModel) - if err != nil { - logs.Error("db_models.UpdateYamlData:", err.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - gpi := &models.GitPackageInfo{ - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - UpdateTime: now, - Status: 0, - GitId: packAgesData[0].GitId, - } - err = db_models.UpdateYamlDetailData(gpi, ormModel) - if err != nil { - logs.Error("db_models.UpdateYamlDetailData", err.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - - tranErr = ormModel.Commit() - if tranErr != nil { - logs.Error(" Commit transaction error:", tranErr.Error()) - continue - } - logs.Info("Transaction committed successfully", k) - } - } else { - logs.Warning("yamlData Is empty, nothing can be done") - } -} - -// ProcGaussYaml -//1. Create a folder -//2. Download yaml -//3. Parse yaml -//4. Save in mysql -func ProcGaussYaml() { - ormModel := orm.NewOrm() - downloadUrls := []string{ - "https://gitee.com/opengauss/openGauss-third_party/raw/master/Third_Party_Open_Source_Software_List.yaml", - "https://gitee.com/opengauss/DataStudio/raw/master/Third_Party_Open_Source_Software_List.yaml", - "https://gitee.com/opengauss/openGauss-connector-jdbc/raw/master/Third_Party_Open_Source_Software_List.yaml", - "https://gitee.com/opengauss/openGauss-tools-sql-translator/raw/master/Third_Party_Open_Source_Software_List.yaml", - } - err := db_models.UpdateYamlOriginMark(ormModel) - if err != nil { - logs.Error("db_models.UpdateYamlOriginMark error:", err.Error()) - return - } - for _, downloadUrl := range downloadUrls { - yamlFileName, err := DownloadGuessYaml(downloadUrl) - if err != nil { - logs.Error("DownloadGuessYaml error:", err.Error()) - return - } - var warehouse string - if len(strings.Split(downloadUrl, "https://gitee.com/opengauss/")) > 1 { - warehouse = strings.Split(strings.Split(downloadUrl, "https://gitee.com/opengauss/")[1], "/")[0] - } - logs.Info("The download yaml file address is:", yamlFileName) - - yamlData, err := util.ParseYaml(yamlFileName) - if err != nil { - logs.Error("util.ParseYaml error:", yamlFileName, err) - return - } - StoreYamlData(yamlData, ormModel, warehouse) - } - err = db_models.DeleteYamlOriginMark(ormModel) - if err != nil { - logs.Error("db_models.DeleteYamlOriginMark error:", err.Error()) - } - DeleteYamlData(ormModel) -} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go deleted file mode 100644 index 9290870..0000000 --- a/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go +++ /dev/null @@ -1,352 +0,0 @@ -package tabletask - -import ( - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "os" - "strings" - "time" - - "cvevulner/common" - "cvevulner/cve-timed-task/db_models" - "cvevulner/cve-timed-task/util" - "cvevulner/models" - - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" - "github.com/astaxie/beego/orm" -) - -// DownloadGuessYaml Download the yaml file of openGauss on Gitee -func DownloadMindSporeYaml() (string, error) { - filePath := beego.AppConfig.String("timedtask::packageDir") + "mindspore_yaml/" - err := util.MakeDir(filePath) - if err != nil { - return "", err - } - yamlFileName := filePath + "mindspore_yaml.yaml" - //download the yaml file - downloadUrl := "https://gitee.com/mindspore/community/raw/master/security/config/Third_Party_Open_Source_Software_List.yaml" - resp, dErr := http.Get(downloadUrl) - defer resp.Body.Close() - if dErr != nil { - logs.Error("get", downloadUrl, "error: ", dErr.Error()) - return "", dErr - } - if resp.StatusCode != http.StatusOK { - logs.Error("get", downloadUrl, resp.Status) - return "", errors.New(resp.Status) - } - saveFile, cErr := os.Create(yamlFileName) - defer saveFile.Close() - if cErr != nil { - logs.Error("Create file error: ", yamlFileName, cErr.Error()) - return "", cErr - } - _, cErr = io.Copy(saveFile, resp.Body) - if cErr != nil { - logs.Error("copy resp.Body to file error", yamlFileName, cErr.Error()) - return "", cErr - } - return yamlFileName, nil -} - -// StoreOpenLookengYaml parse dict -//Store yaml data to mysql -func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string, ormModel orm.Ormer) { - if len(yamlData) > 0 { - err := db_models.UpdateMindYamlOriginMark(ormModel) - if err != nil { - logs.Error("db_models.UpdateYamlOriginMarkLookeng:", err.Error()) - return - } - owner, token := common.GetOwnerAndToken("", 3) - for RepoKey, RepoValue := range yamlData { - for k, v := range RepoValue { - milestones, _ := getMilestone(owner, token, RepoKey) - mile := 0 - if _, ok := v["version"]; !ok || v["version"] == "" { - v["version"] = "" - } - if _, ok := v["handler"]; !ok || v["handler"] == "" { - v["handler"] = "" - } - if _, ok := v["milestone"]; !ok || v["milestone"] == "" { - v["milestone"] = "" - } - for _, milestone := range milestones { - if strings.EqualFold(milestone.Title, v["milestone"]) { - mile = milestone.Id - break - } - } - //open transaction - logs.Info("open transaction") - tranErr := ormModel.Begin() - if tranErr != nil { - logs.Error(" Open transaction error:", tranErr.Error()) - continue - } - if len(v["handler"]) > 0 { - srer, _ := db_models.SelectMindSporeSecurityReviewer(v["handler"], ormModel) - if srer.Id < 1 { - msr := models.Reviewer{NameSpace: v["handler"], Status: 0, OrganizationID: 3} - err = db_models.InsertMindSporeSecurityReviewer(&msr, ormModel) - if err != nil { - logs.Error("db_models.InsertMindSporeSecurityReviewer error:", err.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - } - originData, sErr := db_models.SelectMindYamlOriginData(k, v["version"], RepoKey, ormModel) - now := time.Now() - if sErr != nil { - if errors.Is(orm.ErrNoRows, sErr) { - logs.Warn("error is orm.ErrNoRows, Create a record") - mindSporeYaml := &models.MindSporeYaml{ - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - Status: 1, - CpeName: v["cpeName"], - CreateTime: now.Format("2006-01-02 15:04:05"), - UpdateTime: now.Format("2006-01-02 15:04:05"), - MarkBit: 1, - Repo: RepoKey, - Owner: v["handler"], - Milestone: mile, - } - sErr = db_models.InsertMindYamlOriginData(mindSporeYaml, ormModel) - if sErr != nil { - logs.Error("db_models.InsertMindYamlOriginData error:", sErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - logs.Info("insert MindSporeYaml success") - } else { - logs.Error("db_models.SelectMindYamlOriginData error:", sErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } else { - logs.Info("database records, change the data") - mindSporeYaml := &models.MindSporeYaml{ - Id: originData.Id, - OriginUrl: v["url"], - Status: 1, - CpeName: v["cpeName"], - UpdateTime: now.Format("2006-01-02 15:04:05"), - MarkBit: 1, - Owner: v["handler"], - Milestone: mile, - } - _, sErr = db_models.UpdateMindYamlOriginData(mindSporeYaml, ormModel) - if sErr != nil { - logs.Error("db_models.UpdateMindYamlOriginData:", sErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - packAgesData, rowsAffected, yErr := db_models.SelectYamlData(k, v["version"], ormModel) - if yErr != nil { - logs.Error("db_models.SelectMindYamlData error:", yErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", yErr.Error()) - } - continue - } - if rowsAffected == 0 { - logs.Warn("No record found, Create a record") - var packId int64 - gitOpenEuler, dErr := db_models.SelectYamlLastData(ormModel) - if dErr != nil { - if errors.Is(orm.ErrNoRows, dErr) { - packId = 10000000 - } else { - logs.Error("db_models.SelectYamlLastData error:", dErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } else { - if gitOpenEuler.PackageId >= 10000000 { - packId = gitOpenEuler.PackageId + 1 - } else { - packId = 10000000 - } - } - goe := &models.GitOpenEuler{ - PackageId: packId, - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - CreateTime: now, - UpdateTime: now, - CpePackName: v["cpeName"], - Status: 1, - } - lastId, iErr := models.InsertYamlData(goe) - if iErr != nil { - logs.Error("db_models.InsertYamlData error:", iErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - gpi := &models.GitPackageInfo{ - GitId: lastId, - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - CreateTime: now, - UpdateTime: now, - Decription: "", - Status: 0, - } - iErr = db_models.InsertYamlDetailData(gpi, ormModel) - if iErr != nil { - logs.Error("db_models.InsertYamlDetailData:", iErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } else { - if rowsAffected > 1 { - for _, pv := range packAgesData[1:] { - yErr = db_models.DeleteYamlOpenEulerDetailData(pv.GitId, ormModel) - if yErr != nil { - logs.Error("db_models.DeleteYamlOpenEulerDetailData, error:", yErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - yErr = db_models.DeleteYamlOpenEulerData(pv.GitId, ormModel) - if yErr != nil { - logs.Error("db_models.DeleteYamlOpenEulerData, error:", yErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - } - goe := &models.GitOpenEuler{ - OriginUrl: v["url"], - UpdateTime: now, - CpePackName: v["cpeName"], - Status: 1, - PackageId: packAgesData[0].PackageId, - PackageName: k, - Version: v["version"], - } - yErr = models.UpdateYamlData(goe) - if yErr != nil { - logs.Error("db_models.UpdateYamlData, error:", yErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - gpi := &models.GitPackageInfo{ - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - UpdateTime: now, - Status: 0, - GitId: packAgesData[0].GitId, - } - yErr = db_models.UpdateYamlDetailData(gpi, ormModel) - if yErr != nil { - logs.Error("db_models.UpdateYamlDetailData, error: ", yErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - tranErr = ormModel.Commit() - if tranErr != nil { - logs.Error(" Commit transaction error:", tranErr.Error()) - continue - } - logs.Info("Transaction committed successfully", k) - } - } - err = db_models.DeleteMindYamlOriginMark(ormModel) - if err != nil { - logs.Error("db_models.DeleteMindYamlOriginMark error:", err.Error()) - } - } else { - logs.Warning("yamlData Is empty, nothing can be done") - } -} - -// ProcOpenLookengYaml -//1. Create a folder; -//2. Download yaml; -//3. Parse yaml; -//4. Save in mysql; -func ProcMindSporeYaml() { - ormModel := orm.NewOrm() - yamlFileName, err := DownloadMindSporeYaml() - if err != nil { - logs.Error("DownloadMindSporeYaml error:", err.Error()) - return - } - yamlData, err := util.ParseYamlTypeTwo(yamlFileName) - if err != nil { - logs.Error("util.ParseYamlTypeTwo error: ", yamlFileName, err) - return - } - StoreMindSporeYaml(yamlData, ormModel) - - DeleteYamlData(ormModel) -} - -type miles struct { - Id int - Title string -} - -func getMilestone(owner, token, repo string) (data []miles, _ error) { - url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/milestones?access_token=%s&state=open&sort=due_on&page=1&per_page=100", owner, repo, token) - - do, err := http.Get(url) - if err != nil { - return nil, err - } - - if do.StatusCode > 300 { - return nil, nil - } - - err = json.NewDecoder(do.Body).Decode(&data) - return -} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/parselookeng_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/parselookeng_yaml.go deleted file mode 100644 index fc5ded2..0000000 --- a/cve-vulner-manager/cve-timed-task/tabletask/parselookeng_yaml.go +++ /dev/null @@ -1,287 +0,0 @@ -package tabletask - -import ( - "errors" - "io" - "net/http" - "os" - "time" - - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" - "github.com/astaxie/beego/orm" - - "cvevulner/cve-timed-task/db_models" - "cvevulner/cve-timed-task/util" - "cvevulner/models" -) - -func DownloadOpenlookengYaml() (string, error) { - filePath := beego.AppConfig.String("timedtask::packageDir") + "openlookeng_yaml/" - fErr := util.MakeDir(filePath) - if fErr != nil { - return "", fErr - } - yamlFileName := filePath + "openlookeng_yaml.yaml" - //download the yaml file - downloadUrl := "https://gitee.com/openlookeng/community/raw/master/security/config/Third_Party_Open_Source_Software_List.yaml" - resp, dErr := http.Get(downloadUrl) - if dErr != nil || resp == nil { - return "", dErr - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - logs.Error("get", downloadUrl, resp.Status) - return "", errors.New(resp.Status) - } - saveFile, cErr := os.Create(yamlFileName) - defer saveFile.Close() - if cErr != nil { - logs.Error("Create file error", yamlFileName, cErr.Error()) - return "", cErr - } - _, cErr = io.Copy(saveFile, resp.Body) - if cErr != nil { - logs.Error("copy resp.Body to file error", yamlFileName, cErr.Error()) - return "", cErr - } - return yamlFileName, nil -} - -// StoreOpenLookengYaml parse dict -//Store yaml data to mysql -func StoreOpenlookengYaml(yamlData map[string]map[string]map[string]string, ormModel orm.Ormer) { - if len(yamlData) > 0 { - err := db_models.UpdateOpenlookengYamlOriginMark(ormModel) - if err != nil { - logs.Error("db_models.UpdateYamlOriginMarkLookeng:", err.Error()) - return - } - for RepoKey, RepoValue := range yamlData { - for k, v := range RepoValue { - if _, ok := v["version"]; !ok || v["version"] == "" { - v["version"] = "" - } - //open transaction - logs.Info("open transaction") - tranErr := ormModel.Begin() - if tranErr != nil { - logs.Error(" Open transaction error:", tranErr.Error()) - continue - } - originData, sErr := db_models.SelectOpenlookengYamlOriginData(k, v["version"], RepoKey, ormModel) - now := time.Now() - if sErr != nil { - if errors.Is(orm.ErrNoRows, sErr) { - logs.Warn("error is orm.ErrNoRows, Create a record") - openLookengYaml := &models.OpenLookengYaml{ - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - Status: 1, - CpeName: v["cpeName"], - CreateTime: now.Format("2006-01-02 15:04:05"), - UpdateTime: now.Format("2006-01-02 15:04:05"), - MarkBit: 1, - Repo: RepoKey, - } - sErr = db_models.InsertOpenlookengYamlOriginData(openLookengYaml, ormModel) - if sErr != nil { - logs.Error("db_models.InsertOpenlookengYamlOriginData error:", sErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - logs.Info("insert openLookengYaml success") - } else { - logs.Error("db_models.SelectOpenlookengYamlOriginData error:", sErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } else { - logs.Info("database records, change the data") - openLookengYaml := &models.OpenLookengYaml{ - Id: originData.Id, - OriginUrl: v["url"], - Status: 1, - CpeName: v["cpeName"], - UpdateTime: now.Format("2006-01-02 15:04:05"), - MarkBit: 1, - } - _, sErr = db_models.UpdateOpenlookengYamlOriginData(openLookengYaml, ormModel) - if sErr != nil { - logs.Error("db_models.UpdateOpenlookengYamlOriginData:", sErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - packAgesData, rowsAffected, yErr := db_models.SelectYamlData(k, v["version"], ormModel) - if yErr != nil { - logs.Error("db_models.SelectYamlData error:", yErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - if rowsAffected == 0 { - logs.Warn("No record found, Create a record") - var packId int64 - gitOpenEuler, dErr := db_models.SelectYamlLastData(ormModel) - if dErr != nil { - if errors.Is(orm.ErrNoRows, dErr) { - packId = 10000000 - } else { - logs.Error("db_models.SelectYamlLastData error:", dErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } else { - if gitOpenEuler.PackageId >= 10000000 { - packId = gitOpenEuler.PackageId + 1 - } else { - packId = 10000000 - } - } - goe := &models.GitOpenEuler{ - PackageId: packId, - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - CreateTime: now, - UpdateTime: now, - CpePackName: v["cpeName"], - Status: 1, - } - lastId, iErr := models.InsertYamlData(goe) - if iErr != nil { - logs.Error("db_models.InsertYamlData error:", iErr) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - gpi := &models.GitPackageInfo{ - GitId: lastId, - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - CreateTime: now, - UpdateTime: now, - Decription: "", - Status: 0, - } - iErr = db_models.InsertYamlDetailData(gpi, ormModel) - if iErr != nil { - logs.Error("db_models.InsertYamlDetailData:", iErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } else { - if rowsAffected > 1 { - for _, pv := range packAgesData[1:] { - yErr = db_models.DeleteYamlOpenEulerDetailData(pv.GitId, ormModel) - if yErr != nil { - logs.Error("db_models.DeleteYamlOpenEulerDetailData:", yErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - yErr = db_models.DeleteYamlOpenEulerData(pv.GitId, ormModel) - if yErr != nil { - logs.Error("db_models.DeleteYamlOpenEulerData:", yErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - } - goe := &models.GitOpenEuler{ - OriginUrl: v["url"], - UpdateTime: now, - CpePackName: v["cpeName"], - Status: 1, - PackageId: packAgesData[0].PackageId, - PackageName: k, - Version: v["version"], - } - yErr = models.UpdateYamlData(goe) - if yErr != nil { - logs.Error("db_models.UpdateYamlData:", yErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - gpi := &models.GitPackageInfo{ - PackageName: k, - Version: v["version"], - OriginUrl: v["url"], - UpdateTime: now, - Status: 0, - GitId: packAgesData[0].GitId, - } - yErr = db_models.UpdateYamlDetailData(gpi, ormModel) - if yErr != nil { - logs.Error("db_models.UpdateYamlDetailData", yErr.Error()) - tranErr = ormModel.Rollback() - if tranErr != nil { - logs.Error(" Rollback transaction error:", tranErr.Error()) - } - continue - } - } - tranErr = ormModel.Commit() - if tranErr != nil { - logs.Error(" Commit transaction error:", tranErr.Error()) - continue - } - logs.Info("Transaction committed successfully", k) - } - } - err = db_models.DeleteOpenlookengYamlOriginMark(ormModel) - if err != nil { - logs.Error("db_models.DeleteOpenlookengYamlOriginMark error:", err.Error()) - } - } else { - logs.Warning("yamlData Is empty, nothing can be done") - } - -} - -func ProcOpenlookengYaml() { - ormModel := orm.NewOrm() - yamlFileName, err := DownloadOpenlookengYaml() - if err != nil { - logs.Error("DownloadOpenlookengYaml error:", err.Error()) - return - } - yamlData, err := util.ParseYamlTypeTwo(yamlFileName) - if err != nil { - logs.Error("util.ParseYamlTypeTwo error: ", yamlFileName, err) - return - } - StoreOpenlookengYaml(yamlData, ormModel) - DeleteYamlData(ormModel) -} -- Gitee From abef5e55d105d77d29030317e27bb97401cc9e42 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Tue, 1 Apr 2025 17:24:14 +0800 Subject: [PATCH 2/4] get package from ng --- cve-vulner-manager/conf/product_app.conf | 9 +- cve-vulner-manager/controllers/upload.go | 2 +- .../controller/task_controller.go | 12 +- .../controller/timer_task_controller.go | 7 + .../cve-timed-task/db_models/db_operations.go | 12 +- .../cve-timed-task/tabletask/packages.go | 121 ++++++++++++++++++ cve-vulner-manager/models/modeldb.go | 1 + 7 files changed, 150 insertions(+), 14 deletions(-) create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/packages.go diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 0a135fa..97b437f 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -48,9 +48,7 @@ RunWhitelistTask = "@every 2h" RunPackageCommitterTask = "@every 1h" SupplementCveTask = "0 4 * * *" LongSupplementCveTask = "30 1 * * 0-6" -ParseOpenGaussYamlTask = "30 2 * * 0-6" -ParseMindsporeYamlTask = "30 3 * * 0-6" -ParseOpenlookengYamlTask = "30 4 * * 0-6" +SyncPackages = "45 10 * * 0-6" [mysql] @@ -316,4 +314,7 @@ PRH = 0.50 [unchangedpr] PRN = 0.85 PRL = 0.62 -PRH = 0.27 \ No newline at end of file +PRH = 0.27 + +[cve-manager-ng] +host = http://cve-manager-ng-service.cve-manager-ng:8080 \ No newline at end of file diff --git a/cve-vulner-manager/controllers/upload.go b/cve-vulner-manager/controllers/upload.go index d7d5660..43965a2 100644 --- a/cve-vulner-manager/controllers/upload.go +++ b/cve-vulner-manager/controllers/upload.go @@ -142,7 +142,7 @@ func (u *CveErrorFeedBackController) Get() { } func forwardToCveManagerNG(body []byte) { - ngUrl := "http://cve-manager-ng-service.cve-manager-ng:8080/api/v1/cve/upload" + ngUrl := beego.AppConfig.String("cve-manager-ng::host") + "/api/v1/cve/upload" req, err := http.NewRequest(http.MethodPost, ngUrl, bytes.NewBuffer(body)) if err != nil { logs.Error("new request when forward to ng failed: ", err) diff --git a/cve-vulner-manager/cve-timed-task/controller/task_controller.go b/cve-vulner-manager/cve-timed-task/controller/task_controller.go index 60fcba0..649d2cb 100644 --- a/cve-vulner-manager/cve-timed-task/controller/task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/task_controller.go @@ -22,7 +22,7 @@ func GwController() { } // RunTableTask Parse the artificial CVE table, crawl the CVE official -//website data, and store it in the database +// website data, and store it in the database func RunTableTask() { logs.Info("START TASK:", "Analyze the manual data table, crawl the CVE official website data, and start the timing task") tabletask.AddErrorDetails() @@ -51,7 +51,7 @@ func ToExcelController() { } // RunMappingTask Parse the artificial CVE table, crawl the CVE -//official website data, and store it in the database +// official website data, and store it in the database func RunMappingTask() { logs.Info("START TASK:", "The timing task of parsing the package name mapping table starts") tabletask.ToMysql() @@ -102,10 +102,16 @@ func SupplementCveTask() { } // LongSupplementCveTask Complete the template information of the -//issue with the data on the CVE official website +// issue with the data on the CVE official website func LongSupplementCveTask() { return logs.Info("START TASK:", "long Supplemental cve information task starts") tabletask.LongSupplementCve() logs.Info("END TASK:", "long Supplement cve information task is over") } + +func SyncPackagesTask() { + logs.Info("START TASK:", "SyncPackagesTask task starts") + tabletask.SyncPackages() + logs.Info("END TASK:", "SyncPackagesTask task is over") +} diff --git a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go index c620bbb..9f55a46 100644 --- a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go @@ -86,6 +86,13 @@ func StartTimerTask() error { } logs.Info("Add LongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) + sid, err := c.AddFunc(beego.AppConfig.String("timedtask::SyncPackages"), SyncPackagesTask) + if err != nil { + logs.Error("add SyncPackagesTask error:", err.Error()) + return err + } + logs.Info("add SyncPackagesTask success, id is :", sid) + c.Start() return nil } diff --git a/cve-vulner-manager/cve-timed-task/db_models/db_operations.go b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go index 5db5f47..749f17e 100644 --- a/cve-vulner-manager/cve-timed-task/db_models/db_operations.go +++ b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go @@ -59,10 +59,10 @@ func SelectYamlLastData(ORM orm.Ormer) (models.GitOpenEuler, error) { // InsertYamlData insert data func InsertYamlData(gitOpenEuler *models.GitOpenEuler, ORM orm.Ormer) (int64, error) { sqlString := "insert into cve_git_open_euler(package_id,package_name,version,origin_url," + - "create_time,update_time,cpe_packname,status) values(?,?,?,?,?,?,?,?)" + "create_time,update_time,cpe_packname,status,source) values(?,?,?,?,?,?,?,?,?)" result, err := ORM.Raw(sqlString, gitOpenEuler.PackageId, gitOpenEuler.PackageName, gitOpenEuler.Version, gitOpenEuler.OriginUrl, gitOpenEuler.CreateTime, gitOpenEuler.UpdateTime, - gitOpenEuler.CpePackName, gitOpenEuler.Status).Exec() + gitOpenEuler.CpePackName, gitOpenEuler.Status, gitOpenEuler.Source).Exec() if err != nil { return -1, err } @@ -82,10 +82,10 @@ func InsertYamlDetailData(gitPackageInfo *models.GitPackageInfo, ORM orm.Ormer) return err } -// DeleteYamlOpenEulerDetailData delete origin data -func DeleteYamlOpenEulerDetailData(gitId int64, ORM orm.Ormer) error { - sqlString := "delete from cve_git_package_info where git_id = ?" - _, err := ORM.Raw(sqlString, gitId).Exec() +func DeleteYamlOpenEulerDataBySource(ORM orm.Ormer, source string) error { + sql := "delete from cve_git_open_euler where source = ?" + _, err := ORM.Raw(sql, source).Exec() + return err } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/packages.go b/cve-vulner-manager/cve-timed-task/tabletask/packages.go new file mode 100644 index 0000000..bcad46a --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/packages.go @@ -0,0 +1,121 @@ +package tabletask + +import ( + "errors" + "fmt" + "net/http" + "strings" + "time" + + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/opensourceways/server-common-lib/utils" + + "cvevulner/cve-timed-task/db_models" + "cvevulner/models" +) + +const ( + pageSize = 100 + statusPushed = 1 + sourceNG = "ng" +) + +func SyncPackages() { + pkgs, err := getAllPackages() + if err != nil { + logs.Error("sync packages from ng failed:", err) + return + } + + db := orm.NewOrm() + if err = db.Begin(); err != nil { + logs.Error("sync packages begin transaction failed:", err) + return + } + + if err = db_models.DeleteYamlOpenEulerDataBySource(db, sourceNG); err != nil { + logs.Error("sync packages delete data by source failed:", err) + db.Rollback() + + return + } + + now := time.Now() + for _, v := range pkgs { + goe := models.GitOpenEuler{ + PackageName: v.PackageName, + Version: strings.Join(v.PackageVersion, ","), + CreateTime: now, + UpdateTime: now, + CpePackName: v.CpeName, + Status: statusPushed, + Source: sourceNG, + } + + if _, err = db_models.InsertYamlData(&goe, db); err != nil { + logs.Error("sync packages failed:", goe, err) + } + } + + db.Commit() +} + +type Packages struct { + CpeName string `json:"cpe_name"` + PackageName string `json:"package_name"` + PackageVersion []string `json:"package_version"` +} + +func getAllPackages() ([]Packages, error) { + var pkgs []Packages + page := 1 + for { + result, err := getPackageFromNG(page) + if err != nil { + return nil, err + } + + pkgs = append(pkgs, result...) + + if len(result) < pageSize { + break + } + + page++ + } + + return pkgs, nil +} + +type response struct { + Code string `json:"code"` + Msg string `json:"msg"` + Data struct { + Total int `json:"total"` + List []Packages `json:"list"` + } `json:"data"` +} + +func getPackageFromNG(page int) ([]Packages, error) { + url := fmt.Sprintf("%s/api/v1/packages?page=%d&per_page=%d", + beego.AppConfig.String("cve-manager-ng::host"), page, pageSize) + + cli := utils.NewHttpClient(3) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + var resp response + if _, err = cli.ForwardTo(req, &resp); err != nil { + return nil, err + } + + if resp.Msg != "" { + return nil, errors.New(resp.Msg) + } + + return resp.Data.List, nil +} diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index 627e58a..63e0f6a 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -267,6 +267,7 @@ type GitOpenEuler struct { TableName string `orm:"size(128);column(table_name)" description:"数据库pkginfo下的表名，如：mainline, bringInRely"` CpePackName string `orm:"size(512);column(cpe_packname);null" description:"openEuler对应的cpe包名称(或者影响的包名)"` Status int8 `orm:"default(0);column(status)" description:"0: 代表新数据; 1:已推送; 2:代表已废弃"` + Source string `orm:"default(openeuler);column(source)"` CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` -- Gitee From 267303f5c5fec928be69bbba23cd2ed21354f72a Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Thu, 3 Apr 2025 09:54:20 +0800 Subject: [PATCH 3/4] optimize unaffected --- cve-vulner-manager/taskhandler/excel.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cve-vulner-manager/taskhandler/excel.go b/cve-vulner-manager/taskhandler/excel.go index c167525..f7ef021 100644 --- a/cve-vulner-manager/taskhandler/excel.go +++ b/cve-vulner-manager/taskhandler/excel.go @@ -476,14 +476,20 @@ func UnaffectIssueProc(affectBranch string, cvrfFileList map[string][]string, } // 新数据，检查openeuler评分和向量 - if v.IsIssueWithAnalysisVersion() && v.HasAffected() { - commentFunc := func(content string) { - AddCommentToIssue(content, v.IssueNum, owner, v.Repo, accessToken) + if v.IsIssueWithAnalysisVersion() { + if !v.HasBranch(affectBranch) { + continue } - if !CheckOpenEulerScoreAndVector(&v, commentFunc) { - logs.Error("CheckOpenEulerScoreAndVector of [%s %s] failed", v.Repo, v.IssueNum) - continue + if v.HasAffected() { + commentFunc := func(content string) { + AddCommentToIssue(content, v.IssueNum, owner, v.Repo, accessToken) + } + + if !CheckOpenEulerScoreAndVector(&v, commentFunc) { + logs.Error("CheckOpenEulerScoreAndVector of [%s %s] failed", v.Repo, v.IssueNum) + continue + } } } -- Gitee From 5201a8d657015966e09be553a3c3be22470ee367 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Thu, 3 Apr 2025 10:17:36 +0800 Subject: [PATCH 4/4] fix ci --- cve-vulner-manager/conf/product_app.conf | 2 +- .../cve-timed-task/controller/task_controller.go | 1 + cve-vulner-manager/cve-timed-task/db_models/db_operations.go | 1 + cve-vulner-manager/cve-timed-task/tabletask/packages.go | 5 ++++- 4 files changed, 7 insertions(+), 2 deletions(-) diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 97b437f..af312e9 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -48,7 +48,7 @@ RunWhitelistTask = "@every 2h" RunPackageCommitterTask = "@every 1h" SupplementCveTask = "0 4 * * *" LongSupplementCveTask = "30 1 * * 0-6" -SyncPackages = "45 10 * * 0-6" +SyncPackages = "0 2 * * 0-6" [mysql] diff --git a/cve-vulner-manager/cve-timed-task/controller/task_controller.go b/cve-vulner-manager/cve-timed-task/controller/task_controller.go index 649d2cb..855c883 100644 --- a/cve-vulner-manager/cve-timed-task/controller/task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/task_controller.go @@ -110,6 +110,7 @@ func LongSupplementCveTask() { logs.Info("END TASK:", "long Supplement cve information task is over") } +// SyncPackagesTask sync package task. func SyncPackagesTask() { logs.Info("START TASK:", "SyncPackagesTask task starts") tabletask.SyncPackages() diff --git a/cve-vulner-manager/cve-timed-task/db_models/db_operations.go b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go index 749f17e..2f27b28 100644 --- a/cve-vulner-manager/cve-timed-task/db_models/db_operations.go +++ b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go @@ -82,6 +82,7 @@ func InsertYamlDetailData(gitPackageInfo *models.GitPackageInfo, ORM orm.Ormer) return err } +// DeleteYamlOpenEulerDataBySource clear old data. func DeleteYamlOpenEulerDataBySource(ORM orm.Ormer, source string) error { sql := "delete from cve_git_open_euler where source = ?" _, err := ORM.Raw(sql, source).Exec() diff --git a/cve-vulner-manager/cve-timed-task/tabletask/packages.go b/cve-vulner-manager/cve-timed-task/tabletask/packages.go index bcad46a..f4f43eb 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/packages.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/packages.go @@ -22,6 +22,7 @@ const ( sourceNG = "ng" ) +// SyncPackages sync packages from ng. func SyncPackages() { pkgs, err := getAllPackages() if err != nil { @@ -62,6 +63,7 @@ func SyncPackages() { db.Commit() } +// Packages info about package. type Packages struct { CpeName string `json:"cpe_name"` PackageName string `json:"package_name"` @@ -102,7 +104,8 @@ func getPackageFromNG(page int) ([]Packages, error) { url := fmt.Sprintf("%s/api/v1/packages?page=%d&per_page=%d", beego.AppConfig.String("cve-manager-ng::host"), page, pageSize) - cli := utils.NewHttpClient(3) + const maxRetry = 3 + cli := utils.NewHttpClient(maxRetry) req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, err -- Gitee