diff --git a/conf/app.conf b/conf/app.conf index 4a001ab0e6d294c334cebea12ef8980bcfdcf629..1ca49e1120b5cc50df3cb5e90f7704b056f2204f 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -54,7 +54,7 @@ oricvecheck = 0 16 19 * * * getissueflag = 2 getissue = 0 20 10 * * * issueflag = 2 -createissue = 0 38 15 * * * +createissue = 0 45 19 * * * test = 0/10 * * * * * gittokenflag = 2 issueoath = * * */20 * * * @@ -145,4 +145,9 @@ updateinfo_path = download/updateinfo.xml email_name = "${EMAIL_NAME||***}" email_pwd = "${EMAIL_PWD||***}" email_host = smtp.gmail.com -email_port = 587 \ No newline at end of file +email_port = 587 + +[opengauss] +gauss_owner = opengauss +# git token +git_gauss_token = "${GITEE_GAUSS_TOKEN||xxx}" \ No newline at end of file diff --git a/conf/product_app.conf b/conf/product_app.conf index 12eb12e94030db88f835be40e7852207dd2cf615..7673d6b818f7385df5a33ef05a2053628b9cf5c9 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -61,7 +61,7 @@ gittokenflag = 2 issueoath = * * */20 * * * genexcelflag = 1 genexcel = 0 30 7 * * * -days = -50 +days = -30 prcnum = 50 printlogflag = 1 printlog = 0 0 1 * * 1 @@ -105,11 +105,11 @@ apiurl = https://api.openeuler.org/pkgmanage cveref = https://nvd.nist.gov/vuln/detail/ openeulernum = 3000 # Import cve as data after 2018 -cve_number = 1990 +cve_number = 2018 # Manual data import restrictions cve_number_m = 1990 # Tool import data restriction -cve_number_t = 1990 +cve_number_t = 2018 # Create an issue's warehouse whitelist;1: open; 2: close issue_whitelist = 1 # List of affected branches @@ -138,8 +138,6 @@ sa_timestamp_zone = 28800 updateinfo_path = download/updateinfo.xml [email] - - email_name = "${EMAIL_NAME||***}" email_pwd = "${EMAIL_PWD||***}" email_host = smtp.gmail.com diff --git a/controllers/hook.go b/controllers/hook.go index ef849a01e97e4029aa8463858e19bfc61d8b39cc..784aab978b10436488ccedb3f7dd9fbfff24cd81 100644 --- a/controllers/hook.go +++ b/controllers/hook.go @@ -75,6 +75,7 @@ const ( var comLock sync.Mutex + //HookEventControllers gitee hook callback type HookEventControllers struct { beego.Controller @@ -84,9 +85,13 @@ type HookEventControllers struct { // @router / [post] func (c *HookEventControllers) Post() { if ok := c.isLegitimateHookEvent(); !ok { + c.Ctx.ResponseWriter.WriteHeader(406) + c.Ctx.WriteString("Illegal incident, discarded") return } eventType := c.Ctx.Request.Header.Get(XGIteeEventType) + c.Ctx.ResponseWriter.WriteHeader(200) + c.Ctx.WriteString("Event received: " + eventType) switch eventType { case NoteHookType: //handle comment hook data c.handleNoteDate() diff --git a/models/modeldb.go b/models/modeldb.go index f07503e071d6933c899214a7c5012d7ec6a45242..154fc247462698c6fb333525fcd353de32cb05a9 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -34,21 +34,22 @@ type AdminUser struct { } type VulnCenter struct { - CveId int64 `orm:"pk;auto;column(cve_id)"` - CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` - Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` - CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` - Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求);5:issue已经创建过，不符合要求，不处理"` - CveVersion string `orm:"size(128);column(cve_version)" description:"cve归属版本"` - RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` - PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` - CveUrl string `orm:"size(2048);column(cve_url)" description:"cve下载链接"` - IsExport int8 `orm:"default(0);column(is_export)" description:"0:数据初始化;1:数据已同步到官网；2：数据不需要同步到官网；3：issue已关闭需要同步到官网"` - DataSource int8 `orm:"default(1);column(data_source)" description:"1:软件所提供;2:网络爬取；3:人工处理; 4:兼容之前的数据"` - CveDetailUrl string `orm:"size(1024);column(cve_detail_url)" description:"cve详情链接"` - CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` - UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` - DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` + CveId int64 `orm:"pk;auto;column(cve_id)"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` + CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求);5:issue已经创建过，不符合要求，不处理"` + CveVersion string `orm:"size(128);column(cve_version)" description:"cve归属版本"` + RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + CveUrl string `orm:"size(2048);column(cve_url)" description:"cve下载链接"` + IsExport int8 `orm:"default(0);column(is_export)" description:"0:数据初始化;1:数据已同步到官网；2：数据不需要同步到官网；3：issue已关闭需要同步到官网"` + DataSource int8 `orm:"default(1);column(data_source)" description:"1:软件所提供;2:网络爬取；3:人工处理; 4:兼容之前的数据"` + CveDetailUrl string `orm:"size(1024);column(cve_detail_url)" description:"cve详情链接"` + OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:其他"` + CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` + UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` + DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` } type OpenEulerSA struct { @@ -611,6 +612,7 @@ type OpenGussYaml struct { Version string `orm:"size(64);column(version);index" description:"版本号"` OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` + CpeName string `orm:"size(1024);column(cpe_name)" description:"cpe的名称"` CreateTime string `orm:"size(32);column(create_time)"` UpdateTime string `orm:"size(32);column(update_time);null"` DeleteTime string `orm:"size(32);column(delete_time);null"` diff --git a/task/issuetask.go b/task/issuetask.go index c2d0c3eb50f28111c68148b907c03b06f6687d27..249eed40a948deea57809363002117175a44e51d 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -8,10 +8,12 @@ import ( "github.com/astaxie/beego" "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" + "math/rand" "os" "strconv" "strings" "sync" + "time" ) //GetGiteeToken Get Code Cloud API TOKEN @@ -89,6 +91,8 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears cveData, err := models.QueryIssue(beforeTime, prcnum) if err == nil && len(cveData) > 0 { logs.Info(cveData) + ss := rand.Int31n(10) + time.Sleep(time.Second * time.Duration(ss)) } else { logs.Info("No cve data can be used, current time: ", common.GetCurTime(), ", err: ", err) return err @@ -153,7 +157,7 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) if issueExist { models.UpdateIssueStatus(issueValue, 2) - logs.Info("The cve data has been displayed on the official website, " + + logs.Info("The cve data has been displayed on the official website, "+ "no need to submit an issue on git, cveData: ", issueValue) continue } @@ -214,6 +218,8 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i cveData, err := models.QueryIssueByPackName(issues.PackageName, beforeTime, prcnum) if err == nil && len(cveData) > 0 { logs.Info(cveData) + ss := rand.Int31n(10) + time.Sleep(time.Second * time.Duration(ss)) } else { logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) break @@ -387,8 +393,10 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri } // Query issue template var it models.IssueTemplate - it.CveId = issueValue.CveId - templateErr := models.GetIssueTemplateByColName(&it, "cve_id") + it.CveNum = issueValue.CveNum + it.OwnedVersion = issueValue.CveVersion + it.OwnedComponent = issueValue.PackName + templateErr := models.GetIssueTemplateByColName(&it, "CveNum", "OwnedVersion", "OwnedComponent") if templateErr == nil && it.TemplateId > 0 { it.NVDScore = sr.NVDScore it.NVDVector = sr.NvectorVule @@ -412,7 +420,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri path = it.Repo _, err := taskhandler.UpdateIssueToGit(accessToken, owner, path, issueValue, it) - if err != nil { + if err != nil && err.Error() != "Recreate issue" { logs.Error("更新issue 模板失败， cveId: ", issueValue.CveId, "err: ", err) // Update issue status models.UpdateIssueStatus(issueValue, 3) @@ -476,7 +484,7 @@ func ProcIssue(issueValue models.VulnCenter, accessToken, owner, path string) er } } resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, brandArry) - if err != nil { + if err != nil && err.Error() != "Recreate issue" { logs.Error("创建issue失败, err: ", err, "resp: ", resp, "cveId: ", issueValue.CveId) // Update issue status models.UpdateIssueStatus(issueValue, 3) diff --git a/taskhandler/common.go b/taskhandler/common.go index 09a8680ad909c166bd680039699be980e38e2754..104bbdefa8703f89744118b068b3ae1c7b4ebc3b 100644 --- a/taskhandler/common.go +++ b/taskhandler/common.go @@ -354,7 +354,6 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str its models.IssueTemplate, flag int, issueType, pkgLink string, brandArray []string) string { requestBody := "" scoreType := "" - affectedVersion := "" if sc.ScoreType == "v2" { scoreType = "2.0" } else { @@ -377,6 +376,29 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str if its.CveAnalysis != "" && len(its.CveAnalysis) > 1 { cveAnalysis = strings.Replace(its.CveAnalysis, "\n", "", -1) } + affectedVersion := "" + if its.AffectedVersion != "" && len(its.AffectedVersion) > 0 { + brandsGroup := strings.Split(its.AffectedVersion, ",") + if len(brandsGroup) > 0 { + for i, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brand + "\n" + } + } + } else { + if len(brandArray) > 0 { + for i, brand := range brandArray { + if brand == "" || len(brand) < 2 { + continue + } + affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brand + ":\n" + } + } else { + affectedVersion = "\n" + } + } updateTime := cve.UpdateTime.String() if updateTime != "" && len(updateTime) > 19 { updateTime = updateTime[:19] @@ -401,16 +423,6 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str } } if flag == 1 { - if len(brandArray) > 0 { - for i, brand := range brandArray { - if brand == "" || len(brand) < 2 { - continue - } - affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brand + ":\n" - } - } else { - affectedVersion = "\n" - } if floatOpenEulerScore > 0.0 || (its.OpenEulerVector != "" && len(its.OpenEulerVector) > 1) { cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" nveScore := score + " " + cve.CveLevel @@ -448,18 +460,7 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str "security_hole": "false" }`, accessToken, path, cve.CveNum, StatusName, body, assignee, labels) } - } else if flag == 2 { - if len(brandArray) > 0 { - for i, brand := range brandArray { - if brand == "" || len(brand) < 2 { - continue - } - affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brand + ":\n" - } - } else { - affectedVersion = "\n" - } cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" nveScore := score + " " + cve.CveLevel nveVector := scoreType + "/" + sc.NvectorVule @@ -478,48 +479,15 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str "security_hole": "false" }`, accessToken, path, cve.CveNum, issueType, body, assignee, labels) } else { - if its.AffectedVersion != "" && len(its.AffectedVersion) > 0 { - brandsGroup := strings.Split(its.AffectedVersion, ",") - if len(brandsGroup) > 0 { - for i, brand := range brandsGroup { - if brand == "" || len(brand) < 2 { - continue - } - affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brand + "\n" - } - } - } else { - affectedVersion = "\n" - } if floatOpenEulerScore > 0.0 || (its.OpenEulerVector != "" && len(its.OpenEulerVector) > 1) { - if len(cveAnalysis) > 1 { - cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" - nveScore := score + " " + cve.CveLevel - nveVector := scoreType + "/" + its.NVDVector - oVector := scoreType + "/" + its.OpenEulerVector - body := fmt.Sprintf(bodyUpTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, - cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, - cveAnalysis, openEulerScore, oVector, affectedVersion) - requestBody = fmt.Sprintf(`{ - "access_token": "%s", - "repo": "%s", - "title": "%s", - "state": "%s", - "body": "%s", - "assignee": "%s", - "labels": "%s", - "security_hole": "false" - }`, accessToken, path, cve.CveNum, StatusName, body, its.Assignee, labels) - } else { - cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" - nveScore := score + " " + cve.CveLevel - - nveVector := scoreType + "/" + its.NVDVector - oVector := scoreType + "/" + its.OpenEulerVector - body := fmt.Sprintf(bodyUpTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, - cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, - cveAnalysis, openEulerScore, oVector, affectedVersion) - requestBody = fmt.Sprintf(`{ + cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" + nveScore := score + " " + cve.CveLevel + nveVector := scoreType + "/" + its.NVDVector + oVector := scoreType + "/" + its.OpenEulerVector + body := fmt.Sprintf(bodyUpTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, oVector, affectedVersion) + requestBody = fmt.Sprintf(`{ "access_token": "%s", "repo": "%s", "title": "%s", @@ -529,35 +497,15 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str "labels": "%s", "security_hole": "false" }`, accessToken, path, cve.CveNum, StatusName, body, its.Assignee, labels) - } } else { openEulerScore = " " - if len(cveAnalysis) > 1 { - cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" - nveScore := score + " " + cve.CveLevel - nveVector := scoreType + "/" + sc.NvectorVule - body := fmt.Sprintf(bodyTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, - cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, - cveAnalysis, openEulerScore, affectedVersion) - requestBody = fmt.Sprintf(`{ - "access_token": "%s", - "repo": "%s", - "title": "%s", - "state": "%s", - "body": "%s", - "assignee": "%s", - "labels": "%s", - "security_hole": "false" - }`, accessToken, path, cve.CveNum, StatusName, body, its.Assignee, labels) - } else { - cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" - nveScore := score + " " + cve.CveLevel - - nveVector := scoreType + "/" + sc.NvectorVule - body := fmt.Sprintf(bodyTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, - cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, - cveAnalysis, openEulerScore, affectedVersion) - requestBody = fmt.Sprintf(`{ + cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" + nveScore := score + " " + cve.CveLevel + nveVector := scoreType + "/" + sc.NvectorVule + body := fmt.Sprintf(bodyTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, + cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, + cveAnalysis, openEulerScore, affectedVersion) + requestBody = fmt.Sprintf(`{ "access_token": "%s", "repo": "%s", "title": "%s", @@ -567,7 +515,6 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str "labels": "%s", "security_hole": "false" }`, accessToken, path, cve.CveNum, StatusName, body, its.Assignee, labels) - } } } return requestBody diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index d46be8543a4ecd121813dcbdce2eb5dd28cf3b8a..88c71beb3dcd4d348cc0fb9209f65a2cbfa15030 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -89,7 +89,29 @@ func CreateIssueData(issueTemp *models.IssueTemplate, cve models.VulnCenter, sc return issueTemp } -func CreateIssueToGit(accessToken string, owner string, path string, assignee string, +func GetIssueNum(accessToken, owner, path, issueNum string) error { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/issues/%v?access_token=%v", owner, path, issueNum, accessToken) + if accessToken != "" && owner != "" && path != "" && issueNum != ""{ + issueInfo, err := util.HTTPGetCom(url) + var respBody map[string]interface{} + if err == nil && issueInfo != nil { + err = json.Unmarshal(issueInfo, &respBody) + if err != nil { + logs.Error(err) + return err + } + if respBody !=nil && respBody["number"] != nil && respBody["number"].(string) == issueNum { + return nil + } + } else { + logs.Error("http 请求失败， url: ", url) + return err + } + } + return errors.New("error") +} + +func CreateIssueToGit(accessToken, owner, path, assignee string, cve models.VulnCenter, sc models.Score, brandArray []string) (string, error) { defer common.Catchs() var it models.IssueTemplate @@ -101,6 +123,14 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st it.Assignee = sia.Assignee assignee = sia.Assignee } + if templateErr == nil && it.IssueNum != "" && len(it.IssueNum) > 0 { + issueErr := GetIssueNum(accessToken, owner, path, it.IssueNum) + if issueErr != nil { + models.DeleteIssueTemplate(it.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } + } if templateErr == nil && it.IssueNum != "" && len(it.IssueNum) > 0 { if it.Assignee == "" || len(it.Assignee) == 0 { it.Assignee = assignee @@ -112,7 +142,6 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st } else { labels = beego.AppConfig.String("labelUnFix") } - if accessToken != "" && owner != "" && path != "" { url := "https://gitee.com/api/v5/repos/" + owner + "/issues/" + it.IssueNum score := strconv.FormatFloat(sc.NVDScore, 'f', 1, 64) @@ -277,6 +306,14 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st func UpdateIssueToGit(accessToken string, owner string, path string, cve models.VulnCenter, its models.IssueTemplate) (string, error) { logs.Info("更新模板请求参数: cve: ", cve, ",its: ", its, ", owner: ", owner, ",path: ", path) + if its.IssueNum != "" && len(its.IssueNum) > 0 { + issueErr := GetIssueNum(accessToken, owner, path, its.IssueNum) + if issueErr != nil { + models.DeleteIssueTemplate(its.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } + } //labels := its.IssueLabel sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} specError := models.GetSpecIssueAssignee(&sia, "package_name", "status")