diff --git a/cve-agency-manager/Dockerfile b/cve-agency-manager/Dockerfile index dbe6dc717c4c616d6b4fda3f2bf8e113091330e7..a922a3e3d7f6642de7dee125db301fe2f6256cc6 100644 --- a/cve-agency-manager/Dockerfile +++ b/cve-agency-manager/Dockerfile @@ -14,7 +14,7 @@ COPY ./conf/product.conf /opt/app/conf/app.conf # overwrite config yaml COPY ./cve_tracking /opt/app/cve_tracking RUN chmod 755 -R /opt/app/cve_tracking -COPY --from=BUILDER /go/src/gitee.com/openeuler/cve-agency-manager/cve-agency-manager /opt/app +COPY --from=BUILDER /go/src/gitee.com/openeuler/cve-agency-manager/cve-agency-manager /opt/app/cve-agency-manager WORKDIR /opt/app/ ENTRYPOINT ["/opt/app/cve-agency-manager"] \ No newline at end of file diff --git a/cve-agency-manager/conf/product.conf b/cve-agency-manager/conf/product.conf index 83b4ea927fb021a0f514e65d9933ff4d339ae952..1274e7390137d796433f63d3fc8b90800d489933 100644 --- a/cve-agency-manager/conf/product.conf +++ b/cve-agency-manager/conf/product.conf @@ -7,7 +7,7 @@ EnableDocs = true sqlconn = [log] -log_level = 7 +log_level = 5 log_dir = ./logs log_path = logs/cve.log maxlines=25000 diff --git a/cve-agency-manager/controllers/default.go b/cve-agency-manager/controllers/default.go new file mode 100644 index 0000000000000000000000000000000000000000..9a9e6f2454076ac3c49b8395afd0dea3d1be0d75 --- /dev/null +++ b/cve-agency-manager/controllers/default.go @@ -0,0 +1,15 @@ +package controllers + +import ( + "github.com/astaxie/beego" +) + +type MainController struct { + beego.Controller +} + +func (c *MainController) Get() { + c.Data["Website"] = "beego.me" + c.Data["Email"] = "astaxie@gmail.com" + c.TplName = "index.tpl" +} diff --git a/cve-vulner-manager/controllers/upload.go b/cve-vulner-manager/controllers/upload.go index 41b6f6460cb010e410dfbf730c08e51b82c72c9c..11f6844da1120132ac6fbeaf08faa90ed7b1a728 100644 --- a/cve-vulner-manager/controllers/upload.go +++ b/cve-vulner-manager/controllers/upload.go @@ -184,6 +184,8 @@ func (u *UserUploadController) Post() { for _, CveDataDict := range uploaddata.CveData { defer common.Catchs() logs.Info("Each request parameter: ", CveDataDict) + // Record data flow + AddOrgUpstreamRecord(CveDataDict) var ResData ResultData ids := CveDataDict.Ids if len(ids) < 1 { @@ -350,3 +352,67 @@ func (u *UserUploadController) Post() { resp["body"] = ResDataList return } + +func AddOrgUpstreamRecord(CveDataDict common.CveOriginData) { + orCve := models.OriginUpstreamRecord{} + ids := CveDataDict.Ids + cveNum := CveDataDict.CveNum + updateType := CveDataDict.UpdateType + cvePackName := "" + if len(CveDataDict.CvePackName) > 0 { + cvePackName = strings.Join(CveDataDict.CvePackName, ",") + } + packName := "" + if len(CveDataDict.PackName) > 0 { + packName = strings.Join(CveDataDict.PackName, ",") + } + title := CveDataDict.Title + affectProduct := "" + if len(CveDataDict.AffectProduct) > 0 { + affectProduct = strings.Join(CveDataDict.AffectProduct, ",") + } + cnnvdID := CveDataDict.CnnvdID + cnvdID := CveDataDict.CnvdID + publishedDate := CveDataDict.PublishedDate + vulStatus := CveDataDict.VulStatus + version := CveDataDict.Version + orCve.Ids = ids + if cveNum != "" { + cveNum = common.DeletePreAndSufSpace(cveNum) + } + orCve.CveNum = cveNum + orCve.Version = version + orCve.UpdateType = updateType + orCve.CvePackName = cvePackName + orCve.Credibility = CveDataDict.Credibility + if packName != "" { + packName = common.DeletePreAndSufSpace(packName) + } + orCve.PackName = packName + orCve.Title = title + if affectProduct == "" { + orCve.AffectProduct = packName + } + orCve.CnnvdID = cnnvdID + orCve.CnvdID = cnvdID + orCve.IsExit = 0 + orCve.PublishedDate = publishedDate + orCve.FirstPerTime = CveDataDict.GetTime + orCve.FirstGetTime = CveDataDict.EndGetTime + orCve.VulStatus = vulStatus + if strings.ToLower(updateType) == "delete" { + orCve.Status = 3 + } else if strings.ToLower(updateType) == "update" { + orCve.Status = 1 + } else { + orCve.Status = 0 + } + orCve.CreateTime = common.GetCurTime() + err := models.InsertOriginCveRecord(&orCve) + if err != nil { + logs.Error("InsertOriginCveRecord, err: ", err) + } + // Get the date one month ago + beforeDate := common.GetBeforeDate(1, -30) + models.DeleteOriginCveRecord(beforeDate) +} diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index a15895bd2c9df8b4e22f48b9184975310c17aa52..a66bb24cc36103e89cb58a6fed40ebd101b17ed4 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -903,6 +903,37 @@ type MindSporeBrandTags struct { DeleteTime string `orm:"size(32);column(delete_time);null"` } +type OriginUpstreamRecord struct { + CveId int64 `orm:"pk;auto;column(cve_record_id)"` + Ids string `orm:"size(256);column(cve_un_ids)" description:" 唯一编号,根据此字段去重数据, 唯一识别码，可以填cve编号"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + UpdateType string `orm:"size(32);column(update_type);" description:"数据上传类型：insert, update, delete"` + CvePackName string `orm:"size(512);column(cve_packname);null" description:"Cve在上游对应的包名"` + PackName string `orm:"size(512);column(git_packname);index;null" description:"Cve对应的openEuler包名称(或者影响的包名)"` + Title string `orm:"type(text);column(cve_title);null" description:"标题"` + AffectProduct string `orm:"size(512);column(affect_porduct);null" description:"Cve影响的组件, 对应"` + CnnvdID string `orm:"size(256);column(cnnvd_id);null" description:"Cnnvd_id"` + CnvdID string `orm:"size(256);column(cnvd_id);null" description:"Cnvd_id"` + PublishedDate string `orm:"size(32);column(published_date);null" description:"漏洞发布日期"` + FirstPerTime string `orm:"size(32);column(first_per_time)" description:"cve首次披露时间"` + FirstGetTime string `orm:"size(32);column(first_get_time)" description:"cve受影响后首次感知时间"` + VulStatus string `orm:"size(64);column(vul_status);null" description:"漏洞状态，REJECT, DISPUTED"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求;6:低可信度数据已发邮件"` + AffectedScope string `orm:"size(512);column(affected_scope);null" description:"影响范围推理"` + Version string `orm:"size(64);column(version);index" description:"包对应的版本号"` + AttackLink string `orm:"size(512);column(attack_link);null" description:"攻击链路推理"` + IsExit int8 `orm:"default(1);column(is_exit)" description:"1: 当前包对应在src-openEuler有对应仓库; 0: 无;2:临时值"` + Credibility int `orm:"default(0);column(credibility_level)" description:"0:包名、版本号都是漏洞库原始数据; + 1:包名通过别名匹配、版本号为漏洞库原始数据; + 2:包名版本号都通过漏洞描述获取; + 3:包名通过漏洞描述获取并通过别名匹配、版本号为漏洞描述获取; + 4:包名版本号通过SA获取、版本号为修复版本号、修复版本以下都视为受影响版本; + 5:包名版本号通过SA获取并通过别名匹配、版本号为修复版本号、修复版本以下都视为受影响版本; + 6:版本号未匹配、包名通过以上任意方式匹配、可信度最低"` + OrgData string `orm:"type(text);column(org_data);null" description:"原始数据"` + CreateTime string `orm:"size(32);column(create_time)"` +} + func CreateDb() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -940,7 +971,7 @@ func CreateDb() bool { new(OpenGaussPlatform), new(OpenGaussListTemp), new(OpenEulerRepoOrigin), new(OpenGaussSecurityReviewer), new(MindSporeYaml), new(MindSporeSecurityReviewer), - new(MindSporeBrandTags), + new(MindSporeBrandTags), new(OriginUpstreamRecord), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/cve-vulner-manager/models/uploadcve.go b/cve-vulner-manager/models/uploadcve.go index 8aa2e734bfc6f0d34353cd0dd1db827a5eb28d34..3d4eb0b78eaac3ea5968714ae42b75237e5bd2a4 100644 --- a/cve-vulner-manager/models/uploadcve.go +++ b/cve-vulner-manager/models/uploadcve.go @@ -997,3 +997,16 @@ func GetMindSporeYaml(opy *MindSporeYaml, colName ...string) error { return err } } + +func InsertOriginCveRecord(our *OriginUpstreamRecord) error { + o := orm.NewOrm() + id, err := o.Insert(our) + logs.Info("InsertOriginCveRecord, id: ", id, ", err: ", err) + return err +} + +func DeleteOriginCveRecord(beforeDate string) { + o := orm.NewOrm() + err := o.Raw("delete from cve_origin_upstream_record where create_time < ?", beforeDate).QueryRow() + logs.Info("DeleteOriginCveRecord", err) +} diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go index bfd42abdf5a113720cbfd4062a1f8e5b1837a400..589347323ca4e55e69b35b376b0e864178555364 100644 --- a/cve-vulner-manager/task/issuetask.go +++ b/cve-vulner-manager/task/issuetask.go @@ -248,8 +248,15 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears } } } + var it models.IssueTemplate + it.CveId = issueValue.CveId + it.CveNum = issueValue.CveNum + templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") + if templateErr != nil { + logs.Warn("addUnlimitedIssue, templateErr:", templateErr, ", CveNum: ", issueValue.CveNum) + } // Process each piece of cve data - if issueValue.Status == 0 { + if issueValue.Status == 0 || len(it.IssueNum) < 2 { issueValue.Status = 2 mutex.Lock() err := ProcIssue(issueValue, accessToken, owner) @@ -483,12 +490,12 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) er // Query and modify score sr, err := models.QueryIssueScoreRecord(issueValue.CveId, 0) if err != nil { - logs.Error("ProcUpdateIssue, Failed to query score records, cveId: ", - issueValue.CveId, ", err: ", err) + logs.Error("ProcUpdateIssue, Failed to query score records, CveNum: ", + issueValue.CveNum, ", err: ", err) sc, err := models.QueryIssueScore(issueValue.CveId) if err != nil { logs.Error("ProcUpdateIssue, Failed to get Score, err: ", - err, ", cveId: ", issueValue.CveId) + err, ", CveNum: ", issueValue.CveNum) return errors.New("NVD评分缺失") } sr.NVDScore = sc.NVDScore @@ -533,7 +540,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) er } else { logs.Error("ProcUpdateIssue, Get the repo: owner:", owner, ", path:", path, ", Analyst failed, ", "assErr:", assErr, - ", cveid: ", issueValue.CveId, ", Create an issue without a maintainer") + ", CveNum: ", issueValue.CveNum, ", Create an issue without a maintainer") } } else { it.Assignee = repoMainTainer @@ -553,7 +560,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) er issueValue, it) if err != nil && err.Error() != "Recreate issue" { logs.Error("ProcUpdateIssue, Failed to update issue template, "+ - "cveId: ", issueValue.CveId, "err: ", err) + "CveNum: ", issueValue.CveNum, "err: ", err) // Update issue status models.UpdateIssueStatus(issueValue, 3) return err @@ -565,12 +572,12 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) er templetID, err := models.CreateIssueTemplate(&it) if err != nil { logs.Error("ProcUpdateIssue, Failed to modify issue template, "+ - "cveId: ", issueValue.CveId, ",err: ", err) + "CveNum: ", issueValue.CveNum, ",err: ", err) //return err } models.UpdateIssueScoreRe(issueValue, 1) logs.Info("ProcUpdateIssue, Successfully updated the issue template, "+ - "cveId: ", issueValue.CveId, ", templetID: ", templetID) + "CveNum: ", issueValue.CveNum, ", templetID: ", templetID) } return nil } @@ -585,18 +592,18 @@ func EulerIssue(issueValue models.VulnCenter, accessToken, owner, path, assignee assignee, err = taskhandler.GetCollaboratorInfo(accessToken, owner, path) if assignee == "" { logs.Error("ProcIssue, Get the repo： owner:", owner, ",path:", path, ",Analyst failed", - ", cveid: ", issueValue.CveId, ",Create an issue without a maintainer", ", err:", err) + ", CveNum: ", issueValue.CveNum, ",Create an issue without a maintainer", ", err:", err) } } else { assignee = repoMainTainer } } else { - logs.Error("ProcIssue, Failed to obtain security", ", cveId: ", issueValue.CveId, ", err: ", err) + logs.Error("ProcIssue, Failed to obtain security", ", CveNum: ", issueValue.CveNum, ", err: ", err) return errors.New("CVE描述缺失") } sc, err := models.QueryIssueScore(issueValue.CveId) if err != nil { - logs.Error("ProcIssue, Failed to get Score, err: ", err, ", cveId: ", issueValue.CveId) + logs.Error("ProcIssue, Failed to get Score, err: ", err, ", CveNum: ", issueValue.CveNum) return errors.New("NVD评分缺失") } branchList := make([]string, 0) @@ -635,7 +642,7 @@ func EulerIssue(issueValue models.VulnCenter, accessToken, owner, path, assignee } resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, branchList) if err != nil && err.Error() != "Recreate issue" { - logs.Error("ProcIssue, Failed to create issue, err: ", err, ",resp: ", resp, ",cveId: ", issueValue.CveId) + logs.Error("ProcIssue, Failed to create issue, err: ", err, ",resp: ", resp, ",CveNum: ", issueValue.CveNum) // Update issue status models.UpdateIssueStatus(issueValue, 3) return err @@ -756,19 +763,19 @@ func ProcIssue(issueValue models.VulnCenter, accessToken, owner string) error { if issueValue.OrganizationID == 1 { eulerErr := EulerIssue(issueValue, accessToken, owner, path, assignee) if eulerErr != nil { - logs.Error("EulerIssue, eulerErr: ", eulerErr) + logs.Error("EulerIssue, eulerErr: ", eulerErr, issueValue.CveNum) return eulerErr } } else if issueValue.OrganizationID == 2 { gaussErr := GaussIssue(issueValue, accessToken, owner, path, assignee) if gaussErr != nil { - logs.Error("GaussIssue, gaussErr: ", gaussErr) + logs.Error("GaussIssue, gaussErr: ", gaussErr, issueValue.CveNum) return gaussErr } } else { sporeErr := MindSporeIssue(issueValue, accessToken, owner, path, assignee) if sporeErr != nil { - logs.Error("MindSporeIssue, sporeErr: ", sporeErr) + logs.Error("MindSporeIssue, sporeErr: ", sporeErr, issueValue.CveNum) return sporeErr } } diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index 087d4f7024c07abc1823d3749028c6fcf92146af..204162a38fdec6a14281597ea8c9df2d9acef544 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -143,7 +143,7 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, } } assigneeGite := "" - if it.TemplateId > 0 && len(it.IssueNum) > 1 { + if it.TemplateId > 0 && len(it.IssueNum) > 2 { issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, it.IssueNum) if issueErr != nil { models.DeleteIssueTemplate(it.TemplateId) @@ -160,7 +160,7 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, } else { logs.Error("CreateIssueToGit, GetIssueTemplateByColName, templateErr: ", templateErr, ",it: ", it) } - if it.TemplateId > 0 && len(it.IssueNum) > 1 { + if it.TemplateId > 0 && len(it.IssueNum) > 2 { if it.Assignee == "" || len(it.Assignee) == 0 { it.Assignee = assignee } @@ -247,11 +247,16 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, for _, g := range gi { issueErr, _ := GetGiteeIssue(accessToken, owner, path, g.Number) if issueErr == nil { + models.DeleteIssueTemplate(it.TemplateId) + models.UpdateIssueStatus(cve, 0) return "", errors.New("Recreate issue") } } } } + if it.TemplateId > 0 { + models.DeleteIssueTemplate(it.TemplateId) + } issueType := CIssueType labels := beego.AppConfig.String("labelUnFix") if accessToken != "" && owner != "" && path != "" { @@ -422,7 +427,7 @@ func UpdateIssueToGit(accessToken string, owner string, path string, } } assigneeGite := "" - if its.IssueNum != "" && len(its.IssueNum) > 0 { + if its.IssueNum != "" && len(its.IssueNum) > 2 { issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, its.IssueNum) if issueErr != nil { models.DeleteIssueTemplate(its.TemplateId) @@ -438,6 +443,10 @@ func UpdateIssueToGit(accessToken string, owner string, path string, } } } + } else { + models.DeleteIssueTemplate(its.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") } //labels := its.IssueLabel if cve.OrganizationID == 1 { @@ -468,7 +477,7 @@ func UpdateIssueToGit(accessToken string, owner string, path string, var sc models.Score sc, scok := models.QueryScoreByCveId(cve.CveId) if !scok { - logs.Error("UpdateIssueToGit, Score does not exist, cve: ", cve) + logs.Error("UpdateIssueToGit, Score does not exist, cve: ", cve, cve.CveNum) } if len(assigneeGite) > 1 { its.Assignee = assigneeGite @@ -485,7 +494,7 @@ func UpdateIssueToGit(accessToken string, owner string, path string, return "", errors.New("调用gitee更新issue的接口失败") } if _, ok := resp["id"]; !ok { - logs.Error("UpdateIssueToGit, Failed to create issue, err: ", ok, "url: ", url) + logs.Error("UpdateIssueToGit, Failed to create issue, err: ", ok, "url: ", url, cve.CveNum) return "", errors.New("调用gitee更新issue的接口失败") } // Store security bulletin related information diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index 297715dd7e09c2fefd1965e3dfb728363f9b016e..940dda833a4abfffdc7833161e149f872709265c 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -741,12 +741,12 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) } BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { - logs.Error("GenCveVuler, config init error:", err) + logs.Error("GenCveVuler, config init error:", err, cveData.CveNum) return false, err } years, confOk := BConfig.Int("cve::cve_number_t") if confOk != nil { - logs.Error("GenCveVuler, config cve::cve_number_t, error:", confOk) + logs.Error("GenCveVuler, config cve::cve_number_t, error:", confOk, cveData.CveNum) return false, errors.New("数据错误，暂时不处理") } // Import cve as data after 2018 @@ -865,7 +865,7 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) if !versionFlag && !gaussFlag && !sporeFlag { models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 4) logs.Error("GenCveVuler, The version information corresponds to the error, ", - cveData.PackName, cveData.Version, cveData.CveId) + cveData.PackName, cveData.Version, cveData.CveNum) return false, errors.New("数据错误，暂时不处理") } organizationList := make([]int8, 0)