diff --git a/conf/app.conf b/conf/app.conf index 515578663d57f96ae681b036dc2a4fdf81911135..51b8331796f4d968edd648253c4d9418af95a836 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -25,7 +25,6 @@ fileDir = "download" rpUrl = "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" [mysql] - dbhost = "${DB_URI||127.0.0.1}" dbport = 3306 dbuser = "${DB_USER||root}" @@ -72,6 +71,7 @@ printlog = 0 */20 * * * * #email = 1499273991@qq.com #redirect_uri = http://119.8.126.102:80/v1/issue/oauth/callback # -------jianjun gitee 配置 -------- +#owner = src-openeuler owner = cve-test path = jasper email = 7844966+zhangjianjun_code@user.noreply.gitee.com @@ -101,6 +101,8 @@ cveref = https://nvd.nist.gov/vuln/detail/ openeulernum = 3000 # Import cve as data after 2018 cve_number = 2018 +# Create an issue's warehouse whitelist;1: open; 2: close +issue_whitelist = 2 [reflink] comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual.md diff --git a/conf/product_app.conf b/conf/product_app.conf index 72e9d6a0b4ea5aac3d815825576d8bdcd786b992..459c61b30734742c2e275d21ca2fff05dafc0116 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -60,7 +60,7 @@ genexcelflag = 1 genexcel = 00 00 04 * * * days = -30 prcnum = 50 -printlogflag = 2 +printlogflag = 1 printlog = 0 */10 * * * * @@ -94,6 +94,8 @@ cveref = https://nvd.nist.gov/vuln/detail/ openeulernum = 3000 # Import cve as data after 2018 cve_number = 2018 +# Create an issue's warehouse whitelist;1: open; 2: close +issue_whitelist = 2 [reflink] comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual.md diff --git a/controllers/issue.go b/controllers/issue.go index 2d47c2118bdac33308aac57bf8f879efffae3e88..2f50b0c23dec7cadd8df629593fd112e2d1706e1 100644 --- a/controllers/issue.go +++ b/controllers/issue.go @@ -2,6 +2,7 @@ package controllers import ( "cvevulner/errcode" + "cvevulner/models" "encoding/json" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" @@ -16,6 +17,17 @@ func (c *IssueOathCallbackController) RetData(resp map[string]interface{}) { c.ServeJSON() } +type IssueRepoWhiteData struct { + RepoId int64 `json:"id"` + PackageName string `json:"packageName"` + Version string `json:"version"` + Branchs string `json:"branchs"` + Status int8 `json:"status"` + CreateTime string `json:"createTime"` + UpdateTime string `json:"updateTime"` + DeleteTime string `json:"deleteTime"` +} + // @Title UserLogin // @Description UserLogin // @Param body body models.User true "body for user content" @@ -33,3 +45,61 @@ func (c *IssueOathCallbackController) Post() { logs.Info("登录请求参数：", &req) return } + + +type CveIssueWhiteListController struct { + beego.Controller +} + +func (c *CveIssueWhiteListController) RetData(resp map[string]interface{}) { + c.Data["json"] = resp + c.ServeJSON() +} + + +// @Title Get issuewhitelist +// @Description get packages +// @Param status int true (0,1,2) +// @Success 200 {object} models.IssueRepoWhitelist +// @Failure 403 :status is err +// @router / [get] +func (u *CveIssueWhiteListController) Get() { + req := u.Ctx.Request + addr := req.RemoteAddr + logs.Info("Method: ", req.Method, "客户端请求的：addr: ", addr, "Header: ", req.Header, "body: ", req.Body) + resp := make(map[string]interface{}) + var ird []IssueRepoWhiteData + resp["errno"] = errcode.RecodeUnknowErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeUnknowErr) + resp["body"] = []IssueRepoWhiteData{} + defer u.RetData(resp) + status, err := u.GetInt8("status", 0) + if err != nil { + logs.Error("status, err: ", err, ", status: ", status) + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + return + } + issueWhiteData, issueErr := models.QueryIssueWhitelist(status) + if issueErr == nil && len(issueWhiteData) > 0 { + for _, issues := range issueWhiteData { + var irda IssueRepoWhiteData + irda.CreateTime = issues.CreateTime + irda.Status = issues.Status + irda.Version = issues.Version + irda.PackageName = issues.PackageName + irda.UpdateTime = issues.UpdateTime + irda.RepoId = issues.RepoId + irda.DeleteTime = issues.DeleteTime + irda.Branchs = issues.Branchs + ird = append(ird, irda) + } + resp["body"] = ird + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + } else { + resp["errno"] = errcode.RecodeNodata + resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) + return + } +} \ No newline at end of file diff --git a/doc/excel/packagewhitelist.xlsx b/doc/excel/packagewhitelist.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..1fc7a53c3b41641c9f00853377a5611a6d273ff2 Binary files /dev/null and b/doc/excel/packagewhitelist.xlsx differ diff --git a/doc/md/packwhiteapi.md b/doc/md/packwhiteapi.md new file mode 100644 index 0000000000000000000000000000000000000000..06b0b49e0e7888fd62a04d3f5bcec847ce1b4e28 --- /dev/null +++ b/doc/md/packwhiteapi.md @@ -0,0 +1,46 @@ +#### 1.导入指定仓库创建issue的excel文档模板 + * [excel模板链接](../excel/packagewhitelist.xlsx) +#### 2.仓库创建issue白名单查询接口 + - 接口说明： + +| 说明 | 内容 | +| --- | ---- | +| 请求地址 | https://api.openeuler.org/cve-manager/v1/cve/issue/whitelist | +| 请求类型 | GET | +- 参数： + +| 参数 | 是否必传 | 类型 | 数据类型 | 参数说明 | +| --- | --- | --- | --- | --- | +| status | yes | query | int8 | 返回的数据状态：0：查询全部数据；1：查询正常数据；2：查询已删除数据 | + +- 响应返回： + - 数据返回示例： + ``` + { + "body": [ + { + "id": 1, + "packageName": "firefox", # 包名 + "version": "79.0", # 版本信息 + "branchs": "", # 分支信息 + "status": 1, # 1：正常状态；2：已删除状态 + "createTime": "2020-11-03 12:24:23", + "updateTime": "", + "deleteTime": "" + }, + { + "id": 2, + "packageName": "sudo", + "version": "1.8.27", + "branchs": "", + "status": 1, + "createTime": "2020-11-03 12:24:23", + "updateTime": "", + "deleteTime": "" + } + ], + "errmsg": "成功", + "errno": "200" + } + ``` + diff --git a/models/issue.go b/models/issue.go index 6120afa0923ee6fd4287b6a258f52bb8e2f6ce25..99a04783f23dfce01e82b35d4a685f54e5cc6925 100644 --- a/models/issue.go +++ b/models/issue.go @@ -26,6 +26,50 @@ func QueryIssueCveByNum(cvenum, repoPath string) (GiteOriginIssue, bool) { } } +//QueryIssue query issuewhitelist +func QueryIssueWhitelist(status int8) ([]IssueRepoWhitelist, error) { + o := orm.NewOrm() + var irw []IssueRepoWhitelist + if status == 0 { + num, err := o.Raw("select repo_id,package_name,version,branchs,status,create_time,"+ + "update_time,delete_time"+ + " from cve_issue_repo_whitelist order by repo_id desc").QueryRows(&irw) + if err == nil && num > 0 { + logs.Info("cve_issue_repo_whitelist 查询结果：", irw) + } else { + logs.Info("当前创建issue白名单为空, cur_time:", common.GetCurTime(), "err: ", err) + } + return irw, err + } else { + num, err := o.Raw("select repo_id,package_name,version,branchs,status,create_time,"+ + "update_time,delete_time"+ + " from cve_issue_repo_whitelist where status = ? ", status).QueryRows(&irw) + if err == nil && num > 0 { + logs.Info("cve_issue_repo_whitelist 查询结果：", irw) + } else { + logs.Info("当前创建issue白名单为空, cur_time:", common.GetCurTime(), "err: ", err) + } + return irw, err + } +} + +//QueryIssue query center +func QueryIssueByPackName(packName, version, days string, prcnum int) ([]VulnCenter, error) { + o := orm.NewOrm() + var vc []VulnCenter + num, err := o.Raw("select cve_id,cve_num,cve_desc,cve_version,repair_time,"+ + "pack_name,cve_url,cve_level,data_source,update_time,is_export,cve_detail_url"+ + " from cve_vuln_center where pack_name = ? and cve_version = ? and " + + "update_time >= ? and cve_status in (?, ?) "+ + "order by cve_id asc limit ?", packName, version, days, 0, 1, prcnum).QueryRows(&vc) + if err == nil && num > 0 { + logs.Info("cve_vuln_center 查询结果：", vc, "查询条件： packName: ", packName, ", version: ", version) + } else { + logs.Info("当前无cve,需要提交issue, cur_time:", common.GetCurTime(), "err: ", err) + } + return vc, err +} + //QueryIssue query center func QueryIssue(days string, prcnum int) ([]VulnCenter, error) { o := orm.NewOrm() @@ -178,7 +222,7 @@ func UpdatePackageByCveId(pkgStr string, cveId int64) error { return nil } -func ReplacePackageByCveId(pkgList []string,cveId int64) error { +func ReplacePackageByCveId(pkgList []string, cveId int64) error { //===== 先删除 再修改 ===== sec := struct { SecId int64 @@ -192,20 +236,20 @@ func ReplacePackageByCveId(pkgList []string,cveId int64) error { delPkgSql := `DELETE FROM cve_package WHERE sec_id = ?` _, err = o.Raw(delPkgSql, sec.SecId).Exec() if err != nil { - logs.Error("delete cve_package error:",err) + logs.Error("delete cve_package error:", err) } pkgData := make([]Package, 0) for _, v := range pkgList { - if strings.Trim(v," ")==""{ + if strings.Trim(v, " ") == "" { continue } platform := "source" - if strings.Contains(v,".x86_64."){ - platform = "x86_64" - } else if strings.Contains(v,".aarch64.") || strings.Contains(v,".aarch64."){ + if strings.Contains(v, ".x86_64.") { + platform = "x86_64" + } else if strings.Contains(v, ".aarch64.") || strings.Contains(v, ".aarch64.") { platform = "aarch64" } - pkgUrl := fmt.Sprintf(`https://repo.openeuler.org/openEuler-20.03-LTS/update/%s/Packages/%s`,platform, v) + pkgUrl := fmt.Sprintf(`https://repo.openeuler.org/openEuler-20.03-LTS/update/%s/Packages/%s`, platform, v) pv := Package{SecId: sec.SecId, PackName: v, PackUrl: pkgUrl} pkgData = append(pkgData, pv) } @@ -221,17 +265,51 @@ func QueryPackageByCveId(cveId int64) ([]Package, error) { return res, err } +func InsertIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { + o := orm.NewOrm() + if issTempId, err = o.Insert(it); err == nil && issTempId > 0 { + logs.Info("insert cve_issue_template success, issTempId: ", issTempId, "cveNum: ", it.CveNum) + } else { + logs.Error("insert cve_issue_template failed, cveNum:", it.CveNum, "err: ", err) + return 0, err + } + return issTempId, nil +} + +func UpdateIssueTemplateAll(it *IssueTemplate) (issTempId int64, err error) { + o := orm.NewOrm() + if num, err := o.Update(it); err == nil { + logs.Info("update cve_issue_template success, num: ", num, "cveNum: ", it.CveNum) + } else { + logs.Error("update cve_issue_template failed, cveNum:", it.CveNum, "err: ", err) + return 0, err + } + return it.TemplateId, nil +} + +func DeleteIssueTemplate(issTempId int64) error { + o := orm.NewOrm() + var it = IssueTemplate{TemplateId: issTempId} + id, dErr := o.Delete(&it) + if dErr == nil && id > 0 { + logs.Info("delete cve_issue_template success, issTempId: ", issTempId) + } else { + logs.Error("delete cve_issue_template failed, issTempId: ", issTempId) + } + return dErr +} + func CreateIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { o := orm.NewOrm() var localIt IssueTemplate errx := o.Raw("select *"+ " from cve_issue_template where cve_num = ? and issue_num = ?", it.CveNum, it.IssueNum).QueryRow(&localIt) - if errx != nil || localIt.IssueId == 0{ + if errx != nil || localIt.TemplateId == 0 { var issTempId int64 if issTempId, err = o.Insert(it); err == nil { logs.Info("insert cve_issue_template success, issTempId: ", issTempId, "cveNum: ", it.CveNum) } else { - logs.Error("insert issTempId failed, cveNum:", it.CveNum, "err: ", err) + logs.Error("insert cve_issue_template failed, cveNum:", it.CveNum, "err: ", err) return 0, err } return issTempId, nil @@ -240,7 +318,7 @@ func CreateIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { if num, err := o.Update(it); err == nil { logs.Info("update cve_issue_template success, num: ", num, "cveNum: ", it.CveNum) } else { - logs.Error("update issTempId failed, cveNum:", it.CveNum, "err: ", err) + logs.Error("update cve_issue_template failed, cveNum:", it.CveNum, "err: ", err) return 0, err } return it.TemplateId, nil @@ -374,3 +452,31 @@ func GetIssueTplGroupByRepo() (list []IssueTemplate, err error) { _, err = o.QueryTable("cve_issue_template").GroupBy("repo").All(&list) return } + +func CreateIssueRecord(icr *IssueCreateRecord) (issueRecordId int64, err error) { + o := orm.NewOrm() + var localIcr IssueCreateRecord + errx := o.Raw("select *"+ + " from cve_issue_create_record where cve_id = ? and cve_num = ? and cve_version = ?", + icr.CveId, icr.CveNum, icr.CveVersion).QueryRow(&localIcr) + if errx != nil || localIcr.IssueRecordId == 0 { + var issueRecordId int64 + if issueRecordId, err = o.Insert(icr); err == nil { + logs.Info("insert cve_issue_create_record success, issueRecordId: ", issueRecordId, ", cveNum: ", icr.CveNum) + } else { + logs.Error("insert cve_issue_create_record failed, cveData:", icr, ", err: ", err) + return 0, err + } + return issueRecordId, nil + } else { + icr.IssueRecordId = localIcr.IssueRecordId + if num, err := o.Update(icr); err == nil { + logs.Info("update cve_issue_create_record success, num: ", num, ", cveNum: ", icr.CveNum) + } else { + logs.Error("update cve_issue_create_record failed, cveData:", icr, ", err: ", err) + return 0, err + } + return icr.IssueRecordId, nil + } +} + diff --git a/models/modeldb.go b/models/modeldb.go index ff54b79b781a1b6dd0fcd542a1dc529ab0ea1c6d..e7cd474b9b3833f5b94ead7d31f212a4fe46e523 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -38,7 +38,7 @@ type VulnCenter struct { CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` - Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求)"` CveVersion string `orm:"size(128);column(cve_version)" description:"cve归属版本"` RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` @@ -292,7 +292,7 @@ type OriginUpstream struct { CnvdID string `orm:"size(256);column(cnvd_id);null" description:"Cnvd_id"` PublishedDate string `orm:"size(32);column(published_date);null" description:"漏洞发布日期"` VulStatus string `orm:"size(64);column(vul_status);null" description:"漏洞状态，REJECT, DISPUTED"` - Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：数据已处理"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求"` AffectedScope string `orm:"size(512);column(affected_scope);null" description:"影响范围推理"` Version string `orm:"size(64);column(version);index" description:"包对应的版本号"` AttackLink string `orm:"size(512);column(attack_link);null" description:"攻击链路推理"` @@ -327,7 +327,7 @@ type SecurityReviewer struct { } type IssueAssignee struct { - Id int64 `orm:"pk;auto"` + Id int64 `orm:"pk;auto"` Assignee string `orm:"unique" description:"码云空间地址"` } @@ -572,6 +572,34 @@ type EmailList struct { EmailType int8 `orm:"index;column(email_type);default(1)" description:"1: excel导出列表"` } +type IssueRepoWhitelist struct { + RepoId int64 `orm:"pk;auto;column(repo_id)"` + PackageName string `orm:"column(package_name);size(256);index" description:"包名称"` + Version string `orm:"size(64);column(version);index" description:"版本号"` + Branchs string `orm:"size(512);column(branchs);null" description:"仓库对应的分支"` + Status int8 `orm:"default(1);column(status)" description:"1: 新增; 2:删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type IssueCreateRecord struct { + IssueRecordId int64 `orm:"pk;auto;column(id)"` + CveId int64 `orm:"index;column(cve_id)" description:"VulnCenter 外键"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` + CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` + Status int8 `orm:"default(1);column(status)" description:"1:已创建issue；2：未创建issue；3：创建失败; 4: 已导出"` + CveVersion string `orm:"size(128);column(cve_version)" description:"cve归属版本"` + RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd 评分"` + NvectorVule string `orm:"size(256);column(n_vector_value)" description:"nvd vector 原始值"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + func CreateDb() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -594,6 +622,7 @@ func CreateDb() bool { new(OriginUpstreamFixSuggest), new(OriginUpstreamFixSuggestRefTag), new(OriginUpstreamFixSuggestRef), new(GiteOriginIssue), new(OriginExcel), new(ExportRecord), new(GitRepoGroups), new(GiteRepo), new(GiteRepoMember), new(GiteRepoBranch), new(PackageCpe), new(EmailList), new(IssueAssignee), + new(IssueRepoWhitelist), new(IssueCreateRecord), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/routers/commentsRouter_controllers.go b/routers/commentsRouter_controllers.go index c9f06f00e60c022c3fc051c41f8d0bdbc55e02f1..e2a939601e78e632e68ce78d36da56c67f4f5348 100644 --- a/routers/commentsRouter_controllers.go +++ b/routers/commentsRouter_controllers.go @@ -25,6 +25,15 @@ func init() { Filters: nil, Params: nil}) + beego.GlobalControllerRouter["cvevulner/controllers:CveIssueWhiteListController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:CveIssueWhiteListController"], + beego.ControllerComments{ + Method: "Get", + Router: "/", + AllowHTTPMethods: []string{"get"}, + MethodParams: param.Make(), + Filters: nil, + Params: nil}) + beego.GlobalControllerRouter["cvevulner/controllers:FileController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:FileController"], beego.ControllerComments{ Method: "DownLoadExcelByFileCode", diff --git a/routers/router.go b/routers/router.go index 568a177823038410d8d9d34de9e015773fe19e46..d072cc86fb1b4dd61d17b8e493fe8fb6e5ad3c83 100644 --- a/routers/router.go +++ b/routers/router.go @@ -68,6 +68,11 @@ func init() { &controllers.CveErrorFeedBackController{}, ), ), + beego.NSNamespace("/cve/issue/whitelist", + beego.NSInclude( + &controllers.CveIssueWhiteListController{}, + ), + ), ) beego.AddNamespace(ns) } diff --git a/task/issuetask.go b/task/issuetask.go index 40d62252477cfaa997dbd120e213bad36016c3ea..2e6cf7f28e3df9d94f501f7522a50fa5ace3488f 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -59,6 +59,21 @@ func GetGiteeToken() error { return nil } +// Assemble issue record data +func GenIssueRecordData(icr *models.IssueCreateRecord, issueValue models.VulnCenter, sc models.Score, status int8) { + icr.CveId = issueValue.CveId + icr.CveNum = issueValue.CveNum + icr.Description = issueValue.Description + icr.CveLevel = issueValue.CveLevel + icr.Status = status + icr.CveVersion = issueValue.CveVersion + icr.RepairTime = issueValue.RepairTime + icr.PackName = issueValue.PackName + icr.NVDScore = sc.NVDScore + icr.NvectorVule = sc.NvectorVule + icr.CreateTime = common.GetCurTime() +} + //CreateIssue Create issue func CreateIssue() error { defer common.Catchs() @@ -83,69 +98,201 @@ func CreateIssue() error { logs.Error("config crontab::prcnum error:", ok) return ok } + issueWhitelist, ok := BConfig.Int("cve::issue_whitelist") + if ok != nil { + logs.Error("config cve::issue_whitelist error: ", ok) + return ok + } beforeTime := common.GetBeforeTime(days) - for ; ;{ - cveData, err := models.QueryIssue(beforeTime, prcnum) - if err == nil && len(cveData) > 0 { - logs.Info(cveData) - } else { - logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) - return err - } - accessToken := os.Getenv("GITEE_TOKEN") - if accessToken == "" || len(accessToken) < 1 { - logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) - return err + if issueWhitelist == 1 { + issueWhiteData, issueErr := models.QueryIssueWhitelist(1) + if issueErr == nil && len(issueWhiteData) > 0 { + for _, issues := range issueWhiteData { + for ; ; { + cveData, err := models.QueryIssueByPackName(issues.PackageName, issues.Version, beforeTime, prcnum) + if err == nil && len(cveData) > 0 { + logs.Info(cveData) + } else { + logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) + break + } + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime(), ",err: ", err) + break + } + owner := BConfig.String("gitee::owner") + path := BConfig.String("gitee::path") + for index, issueValue := range cveData { + logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) + // Determine whether the issue has been processed + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝" { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) + continue + } + } + // Import cve as data after 2018 + cveNumList := strings.Split(issueValue.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if cveYears <= years { + icr := models.IssueCreateRecord{} + models.UpdateIssueStatus(issueValue, 4) + logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", + years, ",否则不需要在git上提交issue, cveData: ", issueValue) + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + continue + } + GenIssueRecordData(&icr, issueValue, sc, 2) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } + continue + } + } + } + // Determine whether cve has been processed + issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) + if issueExist { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) + continue + } + // Process each piece of cve data + if issueValue.Status == 0 { + err := ProcIssue(issueValue, accessToken, owner, path) + if err != nil { + logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } else { + err := ProcUpdateIssue(issueValue, accessToken, owner, path) + if err != nil { + logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } + // Collect issue record data + icr := models.IssueCreateRecord{} + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + continue + } + GenIssueRecordData(&icr, issueValue, sc, 1) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } + } + } + } } - owner := BConfig.String("gitee::owner") - path := BConfig.String("gitee::path") - for index, issueValue := range cveData { - logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) - // Determine whether the issue has been processed - goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) - if oks { - if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || - goi.State == "已完成" || goi.State == "已拒绝"{ + } else { + for ; ; { + cveData, err := models.QueryIssue(beforeTime, prcnum) + if err == nil && len(cveData) > 0 { + logs.Info(cveData) + } else { + logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) + return err + } + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) + return err + } + owner := BConfig.String("gitee::owner") + path := BConfig.String("gitee::path") + for index, issueValue := range cveData { + logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) + // Determine whether the issue has been processed + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝" { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) + continue + } + } + // Import cve as data after 2018 + cveNumList := strings.Split(issueValue.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if cveYears <= years { + icr := models.IssueCreateRecord{} + models.UpdateIssueStatus(issueValue, 4) + logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", + years, ",否则不需要在git上提交issue, cveData: ", issueValue) + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + continue + } + GenIssueRecordData(&icr, issueValue, sc, 2) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } + continue + } + } + } + // Determine whether cve has been processed + issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) + if issueExist { models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) + logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) continue } - } - // Import cve as data after 2018 - cveNumList := strings.Split(issueValue.CveNum, "-") - if cveNumList != nil && len(cveNumList) > 1 { - cveYears, yearErr := strconv.Atoi(cveNumList[1]) - if yearErr == nil { - if cveYears <= years { - //models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", years, ",否则不需要在git上提交issue, cveData: ", issueValue) - //continue + // Process each piece of cve data + if issueValue.Status == 0 { + err := ProcIssue(issueValue, accessToken, owner, path) + if err != nil { + logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } else { + err := ProcUpdateIssue(issueValue, accessToken, owner, path) + if err != nil { + logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue } } - } - // Determine whether cve has been processed - issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) - if issueExist { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) - continue - } - // Process each piece of cve data - if issueValue.Status == 0 { - err := ProcIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + // Collect issue record data + icr := models.IssueCreateRecord{} + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil || sc.Id == 0 { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) continue } - } else { - err := ProcUpdateIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) - continue + GenIssueRecordData(&icr, issueValue, sc, 1) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) } } } } + return nil } //ProcUpdateIssue Update issue @@ -169,7 +316,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri gitYaml, ok := models.QueryCveOpeneulerdata(issueValue.PackName, issueValue.CveVersion) if !ok || gitYaml.MainTainer == "" || len(gitYaml.MainTainer) < 1 { assignee, assErr := taskhandler.GetCollaboratorInfo(accessToken, owner, path) - if assignee != "" && len(assignee) > 1{ + if assignee != "" && len(assignee) > 1 { lit.Assignee = assignee //return err } else { diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index df656a311fcc96e8b7532b0b8aa27239f09febd4..4cb80f6af70af831bbd4768010c6e54a67a32014 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -24,37 +24,45 @@ func CreateIssueData(issueTemp *models.IssueTemplate, cve models.VulnCenter, sc issueTemp.NVDVector = sc.NvectorVule issueTemp.CveBrief = cve.Description issueTemp.CveLevel = cve.CveLevel - issueTemp.IssueId = int64(resp["id"].(float64)) - issueTemp.IssueNum = resp["number"].(string) - issueTemp.Assignee = assignee - issueTemp.StatusName = resp["state"].(string) - if strings.ToLower(resp["state"].(string)) == "open" || - resp["state"].(string) == "待办的" || - resp["state"].(string) == "开启的" { - issueTemp.Status = 1 - issueTemp.StatusName = "open" - issueTemp.IssueStatus = 1 - } else if strings.ToLower(resp["state"].(string)) == "started" || - strings.ToLower(resp["state"].(string)) == "progressing" || - strings.ToLower(resp["state"].(string)) == "进行中" { - issueTemp.Status = 2 - issueTemp.StatusName = "progressing" - issueTemp.IssueStatus = 3 - } else if strings.ToLower(resp["state"].(string)) == "closed" || resp["state"].(string) == "已完成" { - issueTemp.Status = 3 - issueTemp.StatusName = "closed" - issueTemp.IssueStatus = 2 - } else { - if strings.ToLower(resp["state"].(string)) == "rejected" || resp["state"].(string) == "已拒绝" { - issueTemp.StatusName = "rejected" - issueTemp.Status = 4 - } - if strings.ToLower(resp["state"].(string)) == "suspended" || resp["state"].(string) == "已挂起" { - issueTemp.StatusName = "suspended" - issueTemp.Status = 5 + if resp != nil && len(resp) > 0 { + issueTemp.IssueId = int64(resp["id"].(float64)) + issueTemp.IssueNum = resp["number"].(string) + issueTemp.StatusName = resp["state"].(string) + if strings.ToLower(resp["state"].(string)) == "open" || + resp["state"].(string) == "待办的" || + resp["state"].(string) == "开启的" { + issueTemp.Status = 1 + issueTemp.StatusName = "open" + issueTemp.IssueStatus = 1 + } else if strings.ToLower(resp["state"].(string)) == "started" || + strings.ToLower(resp["state"].(string)) == "progressing" || + strings.ToLower(resp["state"].(string)) == "进行中" { + issueTemp.Status = 2 + issueTemp.StatusName = "progressing" + issueTemp.IssueStatus = 3 + } else if strings.ToLower(resp["state"].(string)) == "closed" || resp["state"].(string) == "已完成" { + issueTemp.Status = 3 + issueTemp.StatusName = "closed" + issueTemp.IssueStatus = 2 + } else { + if strings.ToLower(resp["state"].(string)) == "rejected" || resp["state"].(string) == "已拒绝" { + issueTemp.StatusName = "rejected" + issueTemp.Status = 4 + } + if strings.ToLower(resp["state"].(string)) == "suspended" || resp["state"].(string) == "已挂起" { + issueTemp.StatusName = "suspended" + issueTemp.Status = 5 + } + issueTemp.IssueStatus = 6 } - issueTemp.IssueStatus = 6 + } else { + issueTemp.IssueId = 0 + issueTemp.IssueNum = "nil" + issueTemp.StatusName = "nil" + issueTemp.Status = 0 } + + issueTemp.Assignee = assignee issueTemp.IssueLabel = labels issueTemp.Owner = owner issueTemp.Repo = path @@ -136,66 +144,81 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st issueType := "CVE和安全问题" labels := beego.AppConfig.String("labelUnFix") if accessToken != "" && owner != "" && path != "" { - url := "https://gitee.com/api/v5/repos/" + owner + "/issues" - score := strconv.FormatFloat(sc.NVDScore, 'f', 1, 64) - requestBody := CreateIssueBody(accessToken, owner, path, assignee, - cve, sc, "", score, labels, its, 2, issueType, "", brandArray) - logs.Info("isssue_body: ", requestBody) - if requestBody != "" && len(requestBody) > 1 { - resp, err := util.HTTPPost(url, requestBody) - if err != nil { - logs.Error("url: ", url, "创建issue失败, cveNum: ", cve.CveNum, "err: ", err) - return "", err - } - if _, ok := resp["id"]; !ok { - logs.Error("创建issue 失败, err: ", ok, "url: ", url) - return "", errors.New("创建issue失败") - } - var issueTemp models.IssueTemplate - CreateIssueData(&issueTemp, cve, sc, resp, path, assignee, issueType, labels, owner) - // Store issue data - issTempID, err := models.CreateIssueTemplate(&issueTemp) - if len(brandArray) > 0 { - var brandArrayTmp []string - for _, brand := range brandArray { - brandArrayTmp = append(brandArrayTmp, brand+":") + var issueTemp models.IssueTemplate + CreateIssueData(&issueTemp, cve, sc, nil, path, assignee, issueType, labels, owner) + // Store issue data + issTempID, templateErr := models.InsertIssueTemplate(&issueTemp) + if templateErr == nil && issTempID > 0 { + url := "https://gitee.com/api/v5/repos/" + owner + "/issues" + score := strconv.FormatFloat(sc.NVDScore, 'f', 1, 64) + requestBody := CreateIssueBody(accessToken, owner, path, assignee, + cve, sc, "", score, labels, its, 2, issueType, "", brandArray) + logs.Info("isssue_body: ", requestBody) + if requestBody != "" && len(requestBody) > 1 { + resp, err := util.HTTPPost(url, requestBody) + if err != nil { + logs.Error("url: ", url, "创建issue失败, cveNum: ", cve.CveNum, "err: ", err) + models.DeleteIssueTemplate(issTempID) + return "", err } - brandStr := strings.Join(brandArrayTmp, ",") - issueTemp.AffectedVersion = brandStr - } - if err != nil { - logs.Error("创建issue 模板的数据失败, cveNum: ", cve.CveNum, "err: ", err) - return "", err - } - logs.Info("创建issue 模板的数据成功, issTempID: ", issTempID, "cveNum: ", cve.CveNum) - // Create issue comment - affectedVersion := "" - if len(brandArray) > 0 { - for i, brand := range brandArray { - if brand == "" || len(brand) < 2 { - continue + if _, ok := resp["id"]; !ok { + logs.Error("创建issue 失败, err: ", ok, "url: ", url) + models.DeleteIssueTemplate(issTempID) + return "", errors.New("创建issue失败") + } + var issueTemps models.IssueTemplate + issueTemps.TemplateId = issTempID + CreateIssueData(&issueTemps, cve, sc, resp, path, assignee, issueType, labels, owner) + // Store issue data + issTempIDx, idxErr := models.UpdateIssueTemplateAll(&issueTemps) + if len(brandArray) > 0 { + var brandArrayTmp []string + for _, brand := range brandArray { + brandArrayTmp = append(brandArrayTmp, brand+":") + } + brandStr := strings.Join(brandArrayTmp, ",") + issueTemp.AffectedVersion = brandStr + } + if idxErr != nil { + logs.Error("创建issue 模板的数据失败, cveNum: ", cve, ",err: ", err) + models.DeleteIssueTemplate(issTempID) + return "", err + } + logs.Info("创建issue 模板的数据成功, issTempID: ", issTempIDx, "cveNum: ", cve.CveNum) + // Create issue comment + affectedVersion := "" + if len(brandArray) > 0 { + for i, brand := range brandArray { + if brand == "" || len(brand) < 2 { + continue + } + affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brand + ":\n" } - affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brand + ":\n" + } else { + affectedVersion = affectedVersion + "\n" } + errx := CreateIssueComment(accessToken, owner, path, assignee, cve, resp, affectedVersion) + logs.Info("issue评论创建结果, err: ", errx) + issueNum := resp["number"].(string) + issueID := int64(resp["id"].(float64)) + err = CreateDepositHooks(accessToken, owner, path, cve, issueNum, issueID) + if err != nil { + logs.Error("创建hooks 失败, cveNum: ", cve.CveNum, "err: ", err) + return "", err + } + logs.Info("创建hooks 成功, cveNum: ", cve.CveNum) + // Update issue status + models.UpdateIssueStatus(cve, 2) + // Update score status + models.UpdateIssueScore(cve, 2) + // Update score record status + models.UpdateIssueScoreRe(cve, 1) } else { - affectedVersion = affectedVersion + "\n" + models.DeleteIssueTemplate(issTempID) } - errx := CreateIssueComment(accessToken, owner, path, assignee, cve, resp, affectedVersion) - logs.Info("issue评论创建结果, err: ", errx) - issueNum := resp["number"].(string) - issueID := int64(resp["id"].(float64)) - err = CreateDepositHooks(accessToken, owner, path, cve, issueNum, issueID) - if err != nil { - logs.Error("创建hooks 失败, cveNum: ", cve.CveNum, "err: ", err) - return "", err - } - logs.Info("创建hooks 成功, cveNum: ", cve.CveNum) - // Update issue status - models.UpdateIssueStatus(cve, 2) - // Update score status - models.UpdateIssueScore(cve, 2) - // Update score record status - models.UpdateIssueScoreRe(cve, 1) + } else { + logs.Error("重复创建issue, cve: ", cve, ", templateErr: ", templateErr) + return "", nil } } } diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 19feec8d8f5915fc368b866893db65590a7e22b5..929541ce73a759ca1a0274ca6ae41bbaba3356b0 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -695,7 +695,7 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) cveYears, yearErr := strconv.Atoi(cveNumList[1]) if yearErr == nil { if cveYears <= years { - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 5) logs.Info("cve: ", cveData.CveNum, ",需要大于： ", years, ",否则不需要在git上提交issue, cveData: ", cveData) return false, errors.New("数据错误，暂时不处理") } @@ -1460,7 +1460,8 @@ func GenCveVulerByIssue(cveData models.GiteOriginIssue, cveRef string, openeuler strings.ToLower(cveData.State) == "progressing" || cveData.State == "进行中" || strings.ToLower(cveData.State) == "started" || cveData.State == "开启的" { lock.Lock() - upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 1, goe, cvd, owner) + // The worker template of the issue that is being processed will no longer be replaced with a new template + upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, cvd, owner) logs.Info(upOk, upError) lock.Unlock() } else {