From 7625cd945a27082ded6850e23d78bcb0d5a70926 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Sat, 27 Jul 2024 17:53:18 +0800 Subject: [PATCH 1/4] modify cronjob for nvd score --- cve-vulner-manager/conf/product_app.conf | 2 +- .../controller/timer_task_controller.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index c3cf125..080c8fb 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -46,7 +46,7 @@ RunDelFileTask = "30 9 * * 0-6" RunIssueRecordTask = "30 7 * * 0-6" RunWhitelistTask = "@every 2h" RunPackageCommitterTask = "@every 1h" -SupplementCveTask = "@every 27m" +SupplementCveTask = "0 4 * * *" LongSupplementCveTask = "30 1 * * 0-6" ParseOpenGaussYamlTask = "30 2 * * 0-6" ParseMindsporeYamlTask = "30 3 * * 0-6" diff --git a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go index 96efc9d..23cd7af 100644 --- a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go @@ -79,12 +79,12 @@ func StartTimerTask() error { } logs.Info("Add SupplementCveTask task success, ID is:", IdSupplementCveTask) - IdLongSupplementCveTask, lErr := c.AddFunc(beego.AppConfig.String("timedtask::LongSupplementCveTask"), LongSupplementCveTask) - if lErr != nil { - logs.Error("add LongSupplementCveTask task error:", lErr.Error()) - return lErr - } - logs.Info("Add LongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) + //IdLongSupplementCveTask, lErr := c.AddFunc(beego.AppConfig.String("timedtask::LongSupplementCveTask"), LongSupplementCveTask) + //if lErr != nil { + // logs.Error("add LongSupplementCveTask task error:", lErr.Error()) + // return lErr + //} + //logs.Info("Add LongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) //Parse opengauss yaml file IdParseOpenGaussYamlTask, pErr := c.AddFunc(beego.AppConfig.String("timedtask::ParseOpenGaussYamlTask"), ParseOpenGaussYamlTask) if pErr != nil { -- Gitee From f76f87ea49ad4b601fd2eea48ef7a2e352188bb4 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Wed, 31 Jul 2024 11:16:34 +0800 Subject: [PATCH 2/4] optimize log --- cve-vulner-manager/routers/new_router.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cve-vulner-manager/routers/new_router.go b/cve-vulner-manager/routers/new_router.go index f127957..ab6e86d 100644 --- a/cve-vulner-manager/routers/new_router.go +++ b/cve-vulner-manager/routers/new_router.go @@ -1,6 +1,7 @@ package routers import ( + "io" "os" "github.com/astaxie/beego" @@ -27,13 +28,17 @@ func Init() { } func InitNewRouter() { + writers := []io.Writer{os.Stdout} file, err := os.OpenFile("./logrus.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) if err == nil { - logrus.SetOutput(file) + writers = append(writers, file) } else { logs.Error("create log file for logrus failed: %s", err.Error()) } + multiWriters := io.MultiWriter(writers...) + logrus.SetOutput(multiWriters) + logBulletin := logrus.WithField("module", "new-security-bulletin") logHotPatchBulletin := logrus.WithField("module", "new-hot-patch-security-bulletin") logColdPatchCveCollect := logrus.WithField("module", "new-cold-patch-cve-collect") -- Gitee From 84ecd2a71fc78005e4ef928e7cf7f2851297e34b Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Thu, 1 Aug 2024 19:57:46 +0800 Subject: [PATCH 3/4] fix timezone for orm --- cve-vulner-manager/models/initdb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cve-vulner-manager/models/initdb.go b/cve-vulner-manager/models/initdb.go index 8d9e890..a605d6b 100644 --- a/cve-vulner-manager/models/initdb.go +++ b/cve-vulner-manager/models/initdb.go @@ -37,7 +37,7 @@ func Initdb() bool { if lerr != nil { maxconn = 3000 } - dns := dbuser + ":" + string(bytes) + "@tcp(" + dbhost + ":" + dbport + ")/" + dbname + "?charset=utf8" + dns := dbuser + ":" + string(bytes) + "@tcp(" + dbhost + ":" + dbport + ")/" + dbname + "?charset=utf8&loc=Local" errx := orm.RegisterDriver("mysql", orm.DRMySQL) if errx != nil { logs.Error("RegisterDriver, orm err: ", errx) -- Gitee From e3c50c435c936f9f4893c6e2c91b541c76d7caa3 Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Fri, 2 Aug 2024 11:36:13 +0800 Subject: [PATCH 4/4] optimize build json body --- .../controller/task_controller.go | 6 ++-- .../controller/timer_task_controller.go | 12 ++++---- cve-vulner-manager/taskhandler/common.go | 29 ++++++++++--------- cve-vulner-manager/taskhandler/createissue.go | 8 ++--- 4 files changed, 29 insertions(+), 26 deletions(-) diff --git a/cve-vulner-manager/cve-timed-task/controller/task_controller.go b/cve-vulner-manager/cve-timed-task/controller/task_controller.go index f6beb6a..9d7345a 100644 --- a/cve-vulner-manager/cve-timed-task/controller/task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/task_controller.go @@ -3,13 +3,14 @@ package controller import ( "time" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "cvevulner/cve-timed-task/deletetask" "cvevulner/cve-timed-task/emailtask" "cvevulner/cve-timed-task/gitwebtask" "cvevulner/cve-timed-task/tabletask" "cvevulner/cve-timed-task/util" - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" ) // GwController Grab warehouse yaml data @@ -103,6 +104,7 @@ func SupplementCveTask() { // LongSupplementCveTask Complete the template information of the //issue with the data on the CVE official website func LongSupplementCveTask() { + return logs.Info("START TASK:", "long Supplemental cve information task starts") tabletask.LongSupplementCve() logs.Info("END TASK:", "long Supplement cve information task is over") diff --git a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go index 23cd7af..96efc9d 100644 --- a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go @@ -79,12 +79,12 @@ func StartTimerTask() error { } logs.Info("Add SupplementCveTask task success, ID is:", IdSupplementCveTask) - //IdLongSupplementCveTask, lErr := c.AddFunc(beego.AppConfig.String("timedtask::LongSupplementCveTask"), LongSupplementCveTask) - //if lErr != nil { - // logs.Error("add LongSupplementCveTask task error:", lErr.Error()) - // return lErr - //} - //logs.Info("Add LongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) + IdLongSupplementCveTask, lErr := c.AddFunc(beego.AppConfig.String("timedtask::LongSupplementCveTask"), LongSupplementCveTask) + if lErr != nil { + logs.Error("add LongSupplementCveTask task error:", lErr.Error()) + return lErr + } + logs.Info("Add LongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) //Parse opengauss yaml file IdParseOpenGaussYamlTask, pErr := c.AddFunc(beego.AppConfig.String("timedtask::ParseOpenGaussYamlTask"), ParseOpenGaussYamlTask) if pErr != nil { diff --git a/cve-vulner-manager/taskhandler/common.go b/cve-vulner-manager/taskhandler/common.go index 07b3a3d..9351c6f 100644 --- a/cve-vulner-manager/taskhandler/common.go +++ b/cve-vulner-manager/taskhandler/common.go @@ -49,6 +49,7 @@ type IssueOptions struct { Token string `json:"access_token"` Repo string `json:"repo"` Title string `json:"title"` + State string `json:"state"` IssueType string `json:"issue_type"` Body string `json:"body"` Assignee string `json:"assignee"` @@ -866,7 +867,8 @@ func AddLabelValue(accessToken, path, issueNum, owner, issueLabel string, flag i func CreateIssueBody(accessToken, owner, path, assignee string, cve models.VulnCenter, sc models.Score, openEulerScore, score, labels string, its models.IssueTemplate, flag int, issueType, pkgLink string, brandArray []string) string { - requestBody := "" + + var issueOption IssueOptions scoreType := "" if sc.ScoreType == "v2" { scoreType = "2.0" @@ -1021,7 +1023,7 @@ func CreateIssueBody(accessToken, owner, path, assignee string, genPatchInfo(cve.CveNum), cveAnalysis, openEulerScore, oVector, affectedVersion) } } - requestBody = fmt.Sprintf(`{"access_token": "%s","repo": "%s","title": "%s","state": "%s","body": "%s","assignee": "%s","labels": "%s","security_hole": "false"}`, accessToken, path, cve.CveNum, StatusName, body, assignee, labels) + issueOption = IssueOptions{Token: accessToken, Repo: path, Title: cve.CveNum, State: StatusName, Body: body, Assignee: assignee, Labels: labels} } else { nveScore := score + " " + cve.CveLevel nveVector := nvdType + "/" + sc.NvectorVule @@ -1036,7 +1038,7 @@ func CreateIssueBody(accessToken, owner, path, assignee string, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl+"\n"+getCveDetail(cve.CveNum)+"\n", commentCmd, holeSource(cve.DataSource), genPatchInfo(cve.CveNum), cveAnalysis, openEulerScore, affectedVersion) } - requestBody = fmt.Sprintf(`{"access_token": "%s","repo": "%s","title": "%s","state": "%s","body": "%s","assignee": "%s","labels": "%s","security_hole": "false"}`, accessToken, path, cve.CveNum, StatusName, body, assignee, labels) + issueOption = IssueOptions{Token: accessToken, Repo: path, Title: cve.CveNum, State: StatusName, Body: body, Assignee: assignee, Labels: labels} } } else if flag == 2 { nveScore := score + " " + cve.CveLevel @@ -1053,25 +1055,22 @@ func CreateIssueBody(accessToken, owner, path, assignee string, genPatchInfo(cve.CveNum), cveAnalysis, openEulerScore, affectedVersion) } - issue := IssueOptions{Token: accessToken, Repo: path, Title: cve.CveNum, IssueType: issueType, Body: body, Assignee: assignee, Labels: labels, SecurityHole: false} + issueOption = IssueOptions{Token: accessToken, Repo: path, Title: cve.CveNum, IssueType: issueType, Body: body, Assignee: assignee, Labels: labels, SecurityHole: false} if cve.OrganizationID == util.MindSpore { mindsporeAssignee := beego.AppConfig.String("mindspore::assignee") - issue.Program = "67813" + issueOption.Program = "67813" if assignee != mindsporeAssignee { - issue.Collaborators = mindsporeAssignee + issueOption.Collaborators = mindsporeAssignee } if mile != 0 { - issue.Milestone = int64(mile) + issueOption.Milestone = int64(mile) } for _, s := range util.DeleteSliceField(brandArray, "master") { - issue.Labels = issue.Labels + "," + s + issueOption.Labels = issueOption.Labels + "," + s } } - - bys, _ := json.Marshal(&issue) - requestBody = string(bys) } else { if floatOpenEulerScore > 0.0 || (its.OpenEulerVector != "" && len(its.OpenEulerVector) > 1) { nveScore := score + " " + cve.CveLevel @@ -1093,7 +1092,7 @@ func CreateIssueBody(accessToken, owner, path, assignee string, genPatchInfo(cve.CveNum), cveAnalysis, openEulerScore, oVector, affectedVersion) } } - requestBody = fmt.Sprintf(`{"access_token": "%s","repo": "%s","title": "%s","state": "%s","body": "%s","assignee": "%s","labels": "%s","security_hole": "false"}`, accessToken, path, cve.CveNum, StatusName, body, its.Assignee, labels) + issueOption = IssueOptions{Token: accessToken, Repo: path, Title: cve.CveNum, State: StatusName, Body: body, Assignee: assignee, Labels: labels} } else { openEulerScore = " " nveScore := score + " " + cve.CveLevel @@ -1108,10 +1107,12 @@ func CreateIssueBody(accessToken, owner, path, assignee string, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl+"\n"+getCveDetail(cve.CveNum)+"\n", commentCmd, holeSource(cve.DataSource), genPatchInfo(cve.CveNum), cveAnalysis, openEulerScore, affectedVersion) } - requestBody = fmt.Sprintf(`{"access_token": "%s","repo": "%s","title": "%s","state": "%s","body": "%s","assignee": "%s","labels": "%s","security_hole": "false"}`, accessToken, path, cve.CveNum, StatusName, body, its.Assignee, labels) + issueOption = IssueOptions{Token: accessToken, Repo: path, Title: cve.CveNum, State: StatusName, Body: body, Assignee: assignee, Labels: labels} } } - return requestBody + + requestBody, _ := json.Marshal(issueOption) + return string(requestBody) } func holeSource(sourceCode int8) string { diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index b761ae2..79435e7 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -250,11 +250,11 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, if err != nil { logs.Error("CreateIssueToGit, Update issue failed, cveNum: ", cve.CveNum, ",err: ", err, ",url: ", url) - return "", errors.New("调用gitee更新issue接口失败") + return "", errors.New("调用gitee创建issue接口失败") } if _, ok := resp["id"]; !ok { logs.Error("CreateIssueToGit, Update issue failed, err: ", ok, ", url: ", url) - return "", errors.New("调用gitee更新issue接口失败") + return "", errors.New("调用gitee创建issue接口失败") } logs.Info("CreateIssueToGit, Update issue successfully, "+ "cveNum: ", cve.CveNum, "issueNum: ", resp["number"].(string)) @@ -560,11 +560,11 @@ func UpdateIssueToGit(accessToken, owner, path string, resp, err := util.HTTPPatch(url, requestBody) if err != nil { logs.Error("UpdateIssueToGit, Update issue failed, cveNum: ", cve.CveNum, "err: ", err) - return "", errors.New("调用gitee更新issue的接口失败") + return "", errors.New("调用gitee更新issue接口失败") } if _, ok := resp["id"]; !ok { logs.Error("UpdateIssueToGit, Failed to create issue, err: ", ok, "url: ", url, cve.CveNum) - return "", errors.New("调用gitee更新issue的接口失败") + return "", errors.New("调用gitee更新issue接口失败") } // Store security bulletin related information var sec models.SecurityNotice -- Gitee