From d752f23eaebbf3ef37c5486235a7e9d72cfaabe4 Mon Sep 17 00:00:00 2001 From: zjm <1076115376@qq.com> Date: Mon, 11 Jul 2022 16:10:04 +0800 Subject: [PATCH] fix create issue and interface --- cve-vulner-manager/controllers/hook.go | 2 +- cve-vulner-manager/controllers/upload.go | 32 ++++++++++++------- cve-vulner-manager/models/cve.go | 23 ++++++++++--- cve-vulner-manager/models/issue.go | 30 +++++++++++++---- cve-vulner-manager/models/uploadcve.go | 20 ++++++++++-- cve-vulner-manager/task/cve.go | 6 ++-- cve-vulner-manager/task/issuetask.go | 18 ++++++++--- cve-vulner-manager/task/synccve.go | 18 ++++++----- cve-vulner-manager/task/urgent.go | 6 ++-- cve-vulner-manager/taskhandler/common.go | 3 -- cve-vulner-manager/taskhandler/createissue.go | 21 ++++++++---- cve-vulner-manager/taskhandler/cve.go | 4 +-- cve-vulner-manager/taskhandler/issue.go | 3 ++ 13 files changed, 132 insertions(+), 54 deletions(-) diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index de08d81..3118ef8 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -1789,7 +1789,7 @@ func PostTriggerGetCve(issueTmp models.IssueTemplate, owner, accessToken, cuAcco return } models.UpdateIssueStatus(vc, 1) - issueErr := task.SyncCreateIssue() + issueErr := task.SyncCreateIssue(nil) if issueErr != nil { logs.Error("SyncCveAndIssue, SyncCreateIssue, err: ", issueErr) } diff --git a/cve-vulner-manager/controllers/upload.go b/cve-vulner-manager/controllers/upload.go index 6b78ba0..af4e1a5 100644 --- a/cve-vulner-manager/controllers/upload.go +++ b/cve-vulner-manager/controllers/upload.go @@ -24,15 +24,17 @@ type ResultData struct { Status int `json:"Status"` } -func (c *UserUploadController) RetData(resp map[string]interface{}) { +func (c *UserUploadController) RetData(resp map[string]interface{}, list []string) { c.Data["json"] = resp c.ServeJSON() // sysnc cve and create issue - synErr := task.SyncCveAndIssue() - if synErr != nil { - logs.Error("SyncCveAndIssue, Sync cve data error, err: ", synErr) - } else { - logs.Info("SyncCveAndIssue, cve data has been synchronized") + if list != nil && len(list) > 0 { + synErr := task.SyncCveAndIssue(list) + if synErr != nil { + logs.Error("SyncCveAndIssue, Sync cve data error, err: ", synErr) + } else { + logs.Info("SyncCveAndIssue, cve data has been synchronized") + } } } @@ -158,13 +160,13 @@ func (u *UserUploadController) Post() { if err != nil { resp["errno"] = errcode.RecodeIpErr resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) - u.RetData(resp) + u.RetData(resp, nil) return } } else { resp["errno"] = errcode.RecodeIpErr resp["errmsg"] = errcode.RecodeText(errcode.RecodeIpErr) - u.RetData(resp) + u.RetData(resp, nil) return } //Judge whether it is legal @@ -173,7 +175,7 @@ func (u *UserUploadController) Post() { resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) resp["body"] = []ResultData{} logs.Error("token request parameter is empty!") - u.RetData(resp) + u.RetData(resp, nil) return } else { // Check token @@ -183,7 +185,7 @@ func (u *UserUploadController) Post() { resp["errmsg"] = errcode.RecodeText(errcode.RecodeSessionErr) resp["body"] = []ResultData{} logs.Error("token verification failed!") - u.RetData(resp) + u.RetData(resp, nil) return } } @@ -192,7 +194,7 @@ func (u *UserUploadController) Post() { resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) resp["body"] = []ResultData{} logs.Error("cve data is empty") - u.RetData(resp) + u.RetData(resp, nil) return } for _, CveDataDict := range uploaddata.CveData { @@ -204,7 +206,13 @@ func (u *UserUploadController) Post() { resp["errno"] = errcode.RecodeOk resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) resp["body"] = ResDataList - u.RetData(resp) + var list = make([]string, 0) + for _, v := range ResDataList { + if v.Status == 0 { + list = append(list, v.CveNum) + } + } + u.RetData(resp, list) return } diff --git a/cve-vulner-manager/models/cve.go b/cve-vulner-manager/models/cve.go index 13c0652..70e40db 100644 --- a/cve-vulner-manager/models/cve.go +++ b/cve-vulner-manager/models/cve.go @@ -120,12 +120,27 @@ func QueryOriginCveIssue(prcnum int) ([]GiteOriginIssue, int64, error) { return gs, num, err } -func QueryOriginCve(days string, prcnum, credibilityLevel int) ([]OriginUpstream, int64, error) { +func QueryOriginCve(days string, prcnum, credibilityLevel int,list []string) ([]OriginUpstream, int64, error) { o := orm.NewOrm() var gs []OriginUpstream - num, err := o.Raw("select * from cve_origin_upstream where cve_status in (?, ?) "+ - "and credibility_level <= ? "+ - "order by cve_id asc limit ?", 0, 1, credibilityLevel, prcnum).QueryRows(&gs) + var num int64 + var err error + if list != nil && len(list) > 0 { + var s string + for _, v := range list { + s += "'" + v + "'," + } + if len(s) > 1 { + s = s[:len(s)-1] + } + num, err = o.Raw(fmt.Sprintf("select * from cve_origin_upstream where cve_num in (%s) "+ + "and credibility_level <= %v and cve_status in (0, 1) "+ + "order by cve_id asc limit %v", s,credibilityLevel, prcnum)).QueryRows(&gs) + } else { + num, err = o.Raw("select * from cve_origin_upstream where cve_status in (?, ?) "+ + "and credibility_level <= ? "+ + "order by cve_id asc limit ?", 0, 1, credibilityLevel, prcnum).QueryRows(&gs) + } if err == nil && num > 0 { logs.Info("QueryOriginCveIssue, cve_origin_upstream, search result: ", num) } else { diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go index 19926fe..98dfee8 100644 --- a/cve-vulner-manager/models/issue.go +++ b/cve-vulner-manager/models/issue.go @@ -1,14 +1,16 @@ package models import ( - "cvevulner/common" - "cvevulner/util" "errors" "fmt" - "github.com/astaxie/beego/logs" - "github.com/astaxie/beego/orm" "strings" "sync" + + "cvevulner/common" + "cvevulner/util" + + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" ) var mutex sync.Mutex @@ -75,11 +77,25 @@ func QueryIssueByPackName(packName, days string, prcnum int) ([]VulnCenter, erro } //QueryIssue query center -func QueryIssue(days string, prcnum int) ([]VulnCenter, error) { +func QueryIssue(days string, prcnum int,list []string) ([]VulnCenter, error) { o := orm.NewOrm() var vc []VulnCenter - num, err := o.Raw("select * from cve_vuln_center where cve_status in (?, ?) "+ - "order by cve_id asc limit ?", 0, 1, prcnum).QueryRows(&vc) + var num int64 + var err error + if list != nil && len(list) > 0 { + var s string + for _, v := range list { + s += "'" + v + "'," + } + if len(s) > 1 { + s = s[:len(s)-1] + } + num, err = o.Raw(fmt.Sprintf("select * from cve_vuln_center where cve_num in (%s) and cve_status in (0, 1) "+ + "order by cve_id asc limit %v", s,prcnum)).QueryRows(&vc) + } else { + num, err = o.Raw("select * from cve_vuln_center where cve_status in (?, ?) "+ + "order by cve_id asc limit ?", 0, 1, prcnum).QueryRows(&vc) + } if err == nil && num > 0 { logs.Info("QueryIssue, cve_vuln_center, search result: ", vc) } else { diff --git a/cve-vulner-manager/models/uploadcve.go b/cve-vulner-manager/models/uploadcve.go index e5d0aa8..9a49418 100644 --- a/cve-vulner-manager/models/uploadcve.go +++ b/cve-vulner-manager/models/uploadcve.go @@ -1015,14 +1015,28 @@ func QueryCveOriginPatchInfo(cveNum string) ([]OriginUpstreamPatch, error) { func QueryCveOrigin(cveNum, typ string) bool { sql := "select * from cve_origin_upstream where (cve_num = ? or cve_un_ids = ?) and update_type = ?" o := orm.NewOrm() - var res []OriginUpstream - _, err := o.Raw(sql, cveNum, cveNum, typ).QueryRows(&res) - if err != nil || len(res) == 0 { + var res OriginUpstream + err := o.Raw(sql, cveNum, cveNum, typ).QueryRow(&res) + if err != nil { return false } return true } +func FilterOldData(cve string) bool { + sql := "select * from cve_origin_upstream where cve_num = ?" + o := orm.NewOrm() + var res OriginUpstream + err := o.Raw(sql, cve).QueryRow(&res) + if err != nil{ + return false + } + if res.Credibility > 3 && res.CveId <= 20557 { + return true + } + return false +} + func QueryCveErrorInfo(issueStatus int8, startDate, endDate string) ([]IssueTemplate, bool) { o := orm.NewOrm() var it []IssueTemplate diff --git a/cve-vulner-manager/task/cve.go b/cve-vulner-manager/task/cve.go index 92a89db..faeda5c 100644 --- a/cve-vulner-manager/task/cve.go +++ b/cve-vulner-manager/task/cve.go @@ -1,9 +1,11 @@ package task import ( + "errors" + "cvevulner/common" "cvevulner/taskhandler" - "errors" + "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" ) @@ -11,7 +13,7 @@ import ( //ProcCveOriginData Process raw data obtained by api func ProcCveOriginData(prcNum, days, credibilityLevel, openeulerNum int, cveRef, owner string) (bool, error) { // Process raw data obtained by api - ok, err := taskhandler.GetCveOriginData(prcNum, days, openeulerNum, credibilityLevel, cveRef) + ok, err := taskhandler.GetCveOriginData(prcNum, days, openeulerNum, credibilityLevel, cveRef, nil) if !ok { logs.Error("ProcCveOriginData, GetCveOriginData, err: ", err) } diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go index 9c0142f..8e51e70 100644 --- a/cve-vulner-manager/task/issuetask.go +++ b/cve-vulner-manager/task/issuetask.go @@ -152,9 +152,9 @@ compareRes: return branchFlag } -func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, flag int) error { +func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, flag int,list []string) error { for ; ; { - cveData, err := models.QueryIssue(beforeTime, prcnum) + cveData, err := models.QueryIssue(beforeTime, prcnum, list) if err == nil && len(cveData) > 0 { logs.Info("cveData: ", cveData) //ss := rand.Int31n(10) @@ -164,6 +164,16 @@ func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, fl return err } for index, issueValue := range cveData { + if models.FilterOldData(issueValue.CveNum) { + var it models.IssueTemplate + it.CveId = issueValue.CveId + it.CveNum = issueValue.CveNum + templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") + if templateErr != nil { + models.UpdateIssueStatus(issueValue, 12) + continue + } + } logs.Info("addUnlimitedIssue, Currently processingļ¼š", index, ",cve data, cveNum: ", issueValue.CveNum) // add mutex lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) @@ -477,7 +487,7 @@ func CreateIssue() error { logs.Info("The execution of the issue task of creating cve ends. . .") return limitErr } else { - unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears, 1) + unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears, 1, nil) logs.Info("CreateIssue, addUnlimitedIssue, err: ", unlimitErr) logs.Info("The execution of the issue task of creating cve ends. . .") return unlimitErr @@ -874,7 +884,7 @@ func EmgCreateIssue() error { logs.Info("The execution of the issue task of creating cve ends. . .") return limitErr } else { - unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears, 1) + unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears, 1, nil) logs.Info("CreateIssue, addUnlimitedIssue, err: ", unlimitErr) logs.Info("The execution of the issue task of creating cve ends. . .") return unlimitErr diff --git a/cve-vulner-manager/task/synccve.go b/cve-vulner-manager/task/synccve.go index d70b101..866422f 100644 --- a/cve-vulner-manager/task/synccve.go +++ b/cve-vulner-manager/task/synccve.go @@ -1,20 +1,22 @@ package task import ( + "errors" + "cvevulner/common" "cvevulner/taskhandler" - "errors" + "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" ) // Sync data to cve vulnerability and create issue -func SyncCveAndIssue() error { - syncErr := SyncCveData() +func SyncCveAndIssue(list []string) error { + syncErr := SyncCveData(list) if syncErr != nil { logs.Error("SyncCveAndIssue, SyncCveData, err: ", syncErr) } - issueErr := SyncCreateIssue() + issueErr := SyncCreateIssue(list) if issueErr != nil { logs.Error("SyncCveAndIssue, SyncCreateIssue, err: ", issueErr) return issueErr @@ -22,7 +24,7 @@ func SyncCveAndIssue() error { return nil } -func SyncCveData() error { +func SyncCveData(list []string) error { defer common.Catchs() logs.Info("Start the task of generating cve raw data into cve library.") // Query the cve to be processed, 1: add; 2: modify @@ -53,14 +55,14 @@ func SyncCveData() error { return cErr } // Process raw data obtained by api - ok, syncErr := taskhandler.GetCveOriginData(prcNum, days, openeulerNum, credibilityLevel, cveRef) + ok, syncErr := taskhandler.GetCveOriginData(prcNum, days, openeulerNum, credibilityLevel, cveRef, list) if !ok || syncErr != nil { logs.Error("ProcCveOriginData, GetCveOriginData, err: ", err) } return syncErr } -func SyncCreateIssue() error { +func SyncCreateIssue(list []string) error { defer common.Catchs() logs.Info("The issue task of creating cve begins. . .") // Query the cve to be processed, 1: add; 2: modify @@ -102,7 +104,7 @@ func SyncCreateIssue() error { logs.Info("The execution of the issue task of creating cve ends. . .") return limitErr } else { - unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears, 1) + unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears, 1, list) logs.Info("CreateIssue, addUnlimitedIssue, err: ", unlimitErr) logs.Info("The execution of the issue task of creating cve ends. . .") return unlimitErr diff --git a/cve-vulner-manager/task/urgent.go b/cve-vulner-manager/task/urgent.go index 7c731c5..d73111e 100644 --- a/cve-vulner-manager/task/urgent.go +++ b/cve-vulner-manager/task/urgent.go @@ -1,9 +1,11 @@ package task import ( + "errors" + "cvevulner/common" "cvevulner/taskhandler" - "errors" + "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" ) @@ -50,7 +52,7 @@ func HandUrgentData() error{ // Synchronize excel data syErr := SynchronizeExcelData() // Create cve data - cErr := SyncCreateIssue() + cErr := SyncCreateIssue(nil) logs.Info(syErr, cErr) return nil } diff --git a/cve-vulner-manager/taskhandler/common.go b/cve-vulner-manager/taskhandler/common.go index 83c1fbe..95b4e8a 100644 --- a/cve-vulner-manager/taskhandler/common.go +++ b/cve-vulner-manager/taskhandler/common.go @@ -1012,9 +1012,6 @@ func CreateIssueBody(accessToken, owner, path, assignee string, } } } else if flag == 2 { - if cve.OrganizationID == 1 && models.QueryCveOrigin(cve.CveNum, "update") { - return "" - } nveScore := score + " " + cve.CveLevel nveVector := scoreType + "/" + sc.NvectorVule openEulerScore = " " diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index 5bf347f..2653e7f 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -1,18 +1,20 @@ package taskhandler import ( - "cvevulner/common" - "cvevulner/models" - "cvevulner/util" "encoding/json" "errors" "fmt" - "github.com/astaxie/beego" - "github.com/astaxie/beego/config" - "github.com/astaxie/beego/logs" "strconv" "strings" "time" + + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" + + "github.com/astaxie/beego" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" ) func CreateIssueData(issueTemp *models.IssueTemplate, cve models.VulnCenter, @@ -327,6 +329,13 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, issueType := CIssueType labels := beego.AppConfig.String("labelUnFix") if accessToken != "" && owner != "" && path != "" { + if models.FilterOldData(cve.CveNum) { + return "",nil + } + if cve.OrganizationID == 1 && models.QueryCveOrigin(cve.CveNum, "update"){ + logs.Info("update cve is not create issue , cveNum :", cve.CveNum) + return "", nil + } var issueTemp models.IssueTemplate issueTemp.CreateTime = time.Now() CreateIssueData(&issueTemp, cve, sc, nil, path, assignee, issueType, labels, owner) diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index 071dc10..cfd1dd7 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -1348,12 +1348,12 @@ func AddExDataToCenter(cveData models.OriginExcel, return true, nil } -func GetCveOriginData(prcnum, days, openeulernum, credibilityLevel int, cveRef string) (bool, error) { +func GetCveOriginData(prcnum, days, openeulernum, credibilityLevel int, cveRef string,list []string) (bool, error) { defer common.Catchs() count := 0 beforeTime := common.GetBeforeTime(days) for { - osx, num, err := models.QueryOriginCve(beforeTime, prcnum, credibilityLevel) + osx, num, err := models.QueryOriginCve(beforeTime, prcnum, credibilityLevel, list) if err != nil || num == 0 { logs.Info("There is currently no cve raw data processing, err: ", err, ", Processing time range > beforetime: ", beforeTime, diff --git a/cve-vulner-manager/taskhandler/issue.go b/cve-vulner-manager/taskhandler/issue.go index 1347029..4ce26f7 100644 --- a/cve-vulner-manager/taskhandler/issue.go +++ b/cve-vulner-manager/taskhandler/issue.go @@ -182,6 +182,9 @@ func CheckOriginUpstreamToCenter(ous []models.OriginUpstream, _, ok := common.FindSliceEm(versionList, packageVersion) if key == packageName && ok { cveRes, cveOk := models.QueryCveByNum(ou.CveNum, repoName, organizationID) + if cveOk && cveRes.Status == 12 { + continue + } if cveOk && cveRes.Status == 2 { ics := models.IssueCommunityStatistics{CveId: cveRes.CveId, CveNum: cveRes.CveNum, Repo: repoName, PackageName: packageName, -- Gitee