diff --git a/cve-vulner-manager/common/common.go b/cve-vulner-manager/common/common.go index 2d771eb90eed1f0fa3a3498313845e2996c9b8c1..cf96777f042fad15d761d81fb60c7c1c919cc92b 100644 --- a/cve-vulner-manager/common/common.go +++ b/cve-vulner-manager/common/common.go @@ -449,3 +449,10 @@ func FindSliceEm(slice []string, val string) (int, bool) { } return -1, false } + +func GetBeforeHourTime(hours time.Duration) string { + now := time.Now() + h, _ := time.ParseDuration("-1h") + beforeHours := now.Add(hours * h).Format(DATE_FORMAT) + return beforeHours +} diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index cc0ab6234fbbda92f8ec5b9a86385f700b17581e..1580892a9ce772423ad739a982a83b7888003979 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -57,7 +57,7 @@ getymal = 0 40 1 * * * eulerymalflag = 1 eulergetymal = 0 0 1 * * * cveflag = 1 -getcve = 0 0 3 * * * +getcve = 0 */10 * * * * oricveflag = 1 oricvecheck = 0 0 2 * * * getissueflag = 1 @@ -65,7 +65,7 @@ getissue = 0 20 1,12 * * * issueflag = 1 createissue = 0 0 6 * * * emergissueflag = 1 -emergcreateissue = 0 */5 * * * * +emergcreateissue = 0 */10 * * * * test = 0/10 * * * * * gittokenflag = 2 issueoath = * * */20 * * * @@ -74,9 +74,9 @@ genexcel = 0 30 7 * * * days = -30 prcnum = 50 printlogflag = 1 -printlog = 0 0 1 * * 2,6 +printlog = 0 0 1 * * 2,4,6 unlockflag = 1 -unlock = 0 0 12 * * * +unlock = 0 */5 * * * * hookflag = 1 prochook = 0 1 8 * * * createhookflag = 2 diff --git a/cve-vulner-manager/controllers/upload.go b/cve-vulner-manager/controllers/upload.go index 8fecb53f5d092159c61d9148799870070e275c69..49ec2db9f471923b202927151a3580c78becf859 100644 --- a/cve-vulner-manager/controllers/upload.go +++ b/cve-vulner-manager/controllers/upload.go @@ -193,176 +193,188 @@ func (u *UserUploadController) Post() { return } for _, CveDataDict := range uploaddata.CveData { - defer common.Catchs() - logs.Info("Each request parameter: ", CveDataDict) - // Record data flow - AddOrgUpstreamRecord(CveDataDict) - var ResData ResultData - ids := CveDataDict.Ids - if len(ids) < 1 { - ResData.CveNum = ids - ResData.Status = 1 - ResDataList = append(ResDataList, ResData) - logs.Error("ids is null, cveDataDict:", CveDataDict) - continue + resDataList := AddOrgUpstream(CveDataDict) + if len(resDataList) > 0 { + ResDataList = append(ResDataList, resDataList...) } - cveNum := CveDataDict.CveNum - if len(cveNum) < 2 { - ResData.CveNum = cveNum - ResData.Status = 1 - ResDataList = append(ResDataList, ResData) - logs.Error("CveNum is null, cveDataDict:", CveDataDict) - continue - } - updateType := CveDataDict.UpdateType - cvePackName := "" - if len(CveDataDict.CvePackName) > 0 { - cvePackName = strings.Join(CveDataDict.CvePackName, ",") - } - packName := "" - if len(CveDataDict.PackName) > 0 { - packName = strings.Join(CveDataDict.PackName, ",") - } else { - ResData.CveNum = cveNum - ResData.Status = 1 - ResDataList = append(ResDataList, ResData) - logs.Error("PackName is null, cveDataDict:", CveDataDict) - continue - } - title := CveDataDict.Title - affectProduct := "" - if len(CveDataDict.AffectProduct) > 0 { - affectProduct = strings.Join(CveDataDict.AffectProduct, ",") - } else { - affectProduct = packName - } - cnnvdID := CveDataDict.CnnvdID - cnvdID := CveDataDict.CnvdID - publishedDate := CveDataDict.PublishedDate - vulStatus := CveDataDict.VulStatus - version := CveDataDict.Version - if version == "" { - version = packName - } - var orCve models.OriginUpstream - if ids != "" { - ids = common.DeletePreAndSufSpace(ids) - } - orCve.Ids = ids - if cveNum != "" { - cveNum = common.DeletePreAndSufSpace(cveNum) - } - orCve.CveNum = cveNum - orCve.Version = version - orCve.UpdateType = updateType - orCve.CvePackName = cvePackName - orCve.Credibility = CveDataDict.Credibility - if packName != "" { - packName = common.DeletePreAndSufSpace(packName) - } - orCve.PackName = packName - orCve.Title = title - if affectProduct == "" { - orCve.AffectProduct = packName + } + resp["errno"] = errcode.RecodeOk + resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) + resp["body"] = ResDataList + u.RetData(resp) + return +} + +func AddOrgUpstream(CveDataDict common.CveOriginData) (ResDataList []ResultData) { + defer common.Catchs() + logs.Info("Each request parameter: ", CveDataDict) + // Record data flow + AddOrgUpstreamRecord(CveDataDict) + var ResData ResultData + ids := CveDataDict.Ids + if len(ids) < 1 { + ResData.CveNum = ids + ResData.Status = 1 + ResDataList = append(ResDataList, ResData) + logs.Error("ids is null, cveDataDict:", CveDataDict) + return + } + cveNum := CveDataDict.CveNum + if len(cveNum) < 2 { + ResData.CveNum = cveNum + ResData.Status = 1 + ResDataList = append(ResDataList, ResData) + logs.Error("CveNum is null, cveDataDict:", CveDataDict) + return + } + updateType := CveDataDict.UpdateType + cvePackName := "" + if len(CveDataDict.CvePackName) > 0 { + cvePackName = strings.Join(CveDataDict.CvePackName, ",") + } + packName := "" + if len(CveDataDict.PackName) > 0 { + packName = strings.Join(CveDataDict.PackName, ",") + } else { + ResData.CveNum = cveNum + ResData.Status = 1 + ResDataList = append(ResDataList, ResData) + logs.Error("PackName is null, cveDataDict:", CveDataDict) + return + } + title := CveDataDict.Title + affectProduct := "" + if len(CveDataDict.AffectProduct) > 0 { + affectProduct = strings.Join(CveDataDict.AffectProduct, ",") + } else { + affectProduct = packName + } + cnnvdID := CveDataDict.CnnvdID + cnvdID := CveDataDict.CnvdID + publishedDate := CveDataDict.PublishedDate + vulStatus := CveDataDict.VulStatus + version := CveDataDict.Version + if version == "" { + version = packName + } + var orCve models.OriginUpstream + if ids != "" { + ids = common.DeletePreAndSufSpace(ids) + } + orCve.Ids = ids + if cveNum != "" { + cveNum = common.DeletePreAndSufSpace(cveNum) + } + orCve.CveNum = cveNum + orCve.Version = version + orCve.UpdateType = updateType + orCve.CvePackName = cvePackName + orCve.Credibility = CveDataDict.Credibility + if packName != "" { + packName = common.DeletePreAndSufSpace(packName) + } + orCve.PackName = packName + orCve.Title = title + if affectProduct == "" { + orCve.AffectProduct = packName + } + orCve.CnnvdID = cnnvdID + orCve.CnvdID = cnvdID + orCve.IsExit = 0 + orCve.PublishedDate = publishedDate + orCve.FirstPerTime = CveDataDict.GetTime + orCve.FirstGetTime = CveDataDict.EndGetTime + checkPackageAndVersion(packName, &orCve) + orCve.VulStatus = vulStatus + if strings.ToLower(updateType) == "delete" { + orCve.Status = 3 + } else if strings.ToLower(updateType) == "update" { + orCve.Status = 1 + } else { + orCve.Status = 0 + } + orCve.CreateTime = common.GetCurTime() + orCve.UpdateTime = common.GetCurTime() + var od models.OriginUpstreamDesc + od.EnDescription = CveDataDict.Description.EnDesc + od.ZhDescription = CveDataDict.Description.ZhDesc + var ous models.OriginUpstreamConfig + ous.Nodes = " " + var osi models.OriginUpstreamImpact + osi.Impact = " " + var osp models.OriginUpstreamPoc + osp.Url = CveDataDict.Poc.Url + osp.Date = CveDataDict.Poc.Date + osp.Dbindex = CveDataDict.Poc.Dbindex + osp.Desc = CveDataDict.Poc.Desc + osp.Path = CveDataDict.Poc.Path + osp.Source = CveDataDict.Poc.Source + var ose models.OriginUpstreamEvent + ose.Date = CveDataDict.Event.Date + ose.Url = CveDataDict.Event.Url + ose.Description = CveDataDict.Event.Description + ose.Title = CveDataDict.Event.Title + var osv models.OriginUpstreamVulType + osv.ZhDesc = CveDataDict.VulType.Zh + osv.EnDesc = CveDataDict.VulType.En + osv.Cwe = CveDataDict.VulType.Cwe + var osf models.OriginUpstreamFixSuggest + osf.Detail = CveDataDict.FixSuggest.Detail + dbCve, ok := models.QueryCveOriginByIds(ids) + if ok { + if orCve.Status != 3 { + orCve.Status = 1 } - orCve.CnnvdID = cnnvdID - orCve.CnvdID = cnvdID - orCve.IsExit = 0 - orCve.PublishedDate = publishedDate - orCve.FirstPerTime = CveDataDict.GetTime - orCve.FirstGetTime = CveDataDict.EndGetTime - packNameList := []string{} - if packName != "" && len(packName) > 0 { - packNameList = strings.Split(packName, ",") + orCve.UpdateTime = common.GetCurTime() + if orCve.Status == 3 { + orCve.DeleteTime = common.GetCurTime() } - if len(packNameList) > 0 { - for _, pk := range packNameList { - pkList := strings.Split(pk, "==") - if len(pkList) == 2 { - var gits models.GitOpenEuler - gits.PackageName = pkList[0] - gits.Version = pkList[1] - gits.Status = 1 - ok := models.QueryCveOpeneulerdata(&gits) - if ok { + logs.Info("The currently inserted data already exists: ", dbCve) + } + _, err := models.CreateOriginCve(CveDataDict, &orCve, &od, &ous, &osi, &osp, &ose, &osv, &osf) + if err == nil { + logs.Info("Cve original data is successfully created CveNum: ", CveDataDict.Ids) + ResData.CveNum = CveDataDict.Ids + ResData.Status = 0 + ResDataList = append(ResDataList, ResData) + } else { + logs.Info("cve creation failed CveNum: ", CveDataDict.Ids) + ResData.CveNum = CveDataDict.Ids + ResData.Status = 1 + ResDataList = append(ResDataList, ResData) + } + return +} + +func checkPackageAndVersion(packName string, orCve *models.OriginUpstream) { + packNameList := []string{} + if packName != "" && len(packName) > 0 { + packNameList = strings.Split(packName, ",") + } + if len(packNameList) > 0 { + for _, pk := range packNameList { + pkList := strings.Split(pk, "==") + if len(pkList) == 2 { + var gits models.GitOpenEuler + gits.PackageName = pkList[0] + gits.Version = pkList[1] + gits.Status = 1 + ok := models.QueryCveOpeneulerdata(&gits) + if ok { + orCve.IsExit = 1 + logs.Info("The data corresponding to src-openEuler is: ", gits) + break + } else { + opy := models.OpenGussYaml{PackageName: pkList[0], Version: pkList[1]} + openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + if openErr == nil && opy.Id > 0 { orCve.IsExit = 1 - logs.Info("The data corresponding to src-openEuler is: ", gits) break - } else { - opy := models.OpenGussYaml{PackageName: pkList[0], Version: pkList[1]} - openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") - if openErr == nil && opy.Id > 0 { - orCve.IsExit = 1 - break - } } } } } - orCve.VulStatus = vulStatus - if strings.ToLower(updateType) == "delete" { - orCve.Status = 3 - } else if strings.ToLower(updateType) == "update" { - orCve.Status = 1 - } else { - orCve.Status = 0 - } - orCve.CreateTime = common.GetCurTime() - orCve.UpdateTime = common.GetCurTime() - var od models.OriginUpstreamDesc - od.EnDescription = CveDataDict.Description.EnDesc - od.ZhDescription = CveDataDict.Description.ZhDesc - var ous models.OriginUpstreamConfig - ous.Nodes = " " - var osi models.OriginUpstreamImpact - osi.Impact = " " - var osp models.OriginUpstreamPoc - osp.Url = CveDataDict.Poc.Url - osp.Date = CveDataDict.Poc.Date - osp.Dbindex = CveDataDict.Poc.Dbindex - osp.Desc = CveDataDict.Poc.Desc - osp.Path = CveDataDict.Poc.Path - osp.Source = CveDataDict.Poc.Source - var ose models.OriginUpstreamEvent - ose.Date = CveDataDict.Event.Date - ose.Url = CveDataDict.Event.Url - ose.Description = CveDataDict.Event.Description - ose.Title = CveDataDict.Event.Title - var osv models.OriginUpstreamVulType - osv.ZhDesc = CveDataDict.VulType.Zh - osv.EnDesc = CveDataDict.VulType.En - osv.Cwe = CveDataDict.VulType.Cwe - var osf models.OriginUpstreamFixSuggest - osf.Detail = CveDataDict.FixSuggest.Detail - dbCve, ok := models.QueryCveOriginByIds(ids) - if ok { - if orCve.Status != 3 { - orCve.Status = 1 - } - orCve.UpdateTime = common.GetCurTime() - if orCve.Status == 3 { - orCve.DeleteTime = common.GetCurTime() - } - logs.Info("The currently inserted data already exists: ", dbCve) - } - _, err := models.CreateOriginCve(CveDataDict, &orCve, &od, &ous, &osi, &osp, &ose, &osv, &osf) - if err == nil { - logs.Info("Cve original data is successfully created CveNum: ", CveDataDict.Ids) - ResData.CveNum = CveDataDict.Ids - ResData.Status = 0 - ResDataList = append(ResDataList, ResData) - } else { - logs.Info("cve creation failed CveNum: ", CveDataDict.Ids) - ResData.CveNum = CveDataDict.Ids - ResData.Status = 1 - ResDataList = append(ResDataList, ResData) - } } - resp["errno"] = errcode.RecodeOk - resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) - resp["body"] = ResDataList - u.RetData(resp) - return } func AddOrgUpstreamRecord(CveDataDict common.CveOriginData) { @@ -425,6 +437,6 @@ func AddOrgUpstreamRecord(CveDataDict common.CveOriginData) { logs.Error("InsertOriginCveRecord, err: ", err) } // Get the date one month ago - beforeDate := common.GetBeforeDate(1, -30) + beforeDate := common.GetBeforeDate(1, -90) models.DeleteOriginCveRecord(beforeDate) } diff --git a/cve-vulner-manager/models/cve.go b/cve-vulner-manager/models/cve.go index 3d5d7b71412ac75a0844a10595d10afa6440d6f0..a4fcd02e12bc357b29f0fb9b7e154055b8de763a 100644 --- a/cve-vulner-manager/models/cve.go +++ b/cve-vulner-manager/models/cve.go @@ -58,10 +58,11 @@ func LockUpdateCveIssueStatus(status int, issueId int64) (bool) { } } -func UnLockUpdateCveIssueStatus(beforeStatus, afterStatus int8, updateTime string) (bool) { +func UnLockUpdateCveIssueStatus(beforeStatus, afterStatus int8, updateTime, beforeTime string) (bool) { o := orm.NewOrm() - res, err := o.Raw("UPDATE cve_gite_origin_issue SET proc_status = ?, issue_update_at = ? where proc_status = ?", - afterStatus, updateTime, beforeStatus).Exec() + res, err := o.Raw("UPDATE cve_gite_origin_issue SET proc_status = ?, "+ + "issue_update_at = ? where proc_status = ? and issue_update_at < ?", + afterStatus, updateTime, beforeStatus, beforeTime).Exec() if err == nil { num, _ := res.RowsAffected() if num > 0 { @@ -107,9 +108,9 @@ func QueryOriginCveIssue(prcnum int) ([]GiteOriginIssue, int64, error) { func QueryOriginCve(days string, prcnum, credibilityLevel int) ([]OriginUpstream, int64, error) { o := orm.NewOrm() var gs []OriginUpstream - num, err := o.Raw("select * from cve_origin_upstream where update_time >= ? and cve_status in (?, ?) "+ - "and is_exit = ? and credibility_level <= ? "+ - "order by cve_id asc limit ?", days, 0, 1, 1, credibilityLevel, prcnum).QueryRows(&gs) + num, err := o.Raw("select * from cve_origin_upstream where cve_status in (?, ?) "+ + "and credibility_level <= ? "+ + "order by cve_id asc limit ?", 0, 1, credibilityLevel, prcnum).QueryRows(&gs) if err == nil && num > 0 { logs.Info("QueryOriginCveIssue, cve_origin_upstream, search result: ", num) } else { @@ -204,7 +205,7 @@ func QueryCveByPackName(cveNum, packName string, organizateId int8) (vc []VulnCe return } -func QueryCveByNum(cveNum, packName, version string, organizateId int8) (VulnCenter, bool) { +func QueryCveByNum(cveNum, packName string, organizateId int8) (VulnCenter, bool) { o := orm.NewOrm() var cve VulnCenter err := o.Raw("select * from cve_vuln_center where cve_num = ? and "+ @@ -658,6 +659,19 @@ func QueryOpenSaById(cveId int64) (OpenEulerSA, error) { return os, err } +func QueryOriginUpstreamRecord(our *OriginUpstreamRecord, ids, cveNum string) (error) { + o := orm.NewOrm() + err := o.Raw("select * from cve_origin_upstream_record where cve_un_ids = ? "+ + "and cve_num = ? order by cve_record_id desc limit 1", ids, cveNum).QueryRow(our) + return err +} + +func UpdateOriginUpstreamRecord(ogc *OriginUpstreamRecord, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(ogc, fields...) + return err +} + func UpdateOriginStatus(updateTime, pakName, version string, cveId int64, status int) bool { o := orm.NewOrm() res, err := o.Raw("UPDATE cve_origin_upstream SET "+ @@ -678,12 +692,12 @@ func UpdateOriginStatus(updateTime, pakName, version string, cveId int64, status } } -func LockUpdateOriginStatus(updateTime, pakName, version string, cveId int64, status int) bool { +func LockUpdateOriginStatus(updateTime, pakName, version string, cveId int64, befStatus int8, status int) bool { o := orm.NewOrm() res, err := o.Raw("UPDATE cve_origin_upstream SET "+ "cve_status = ?, update_time = ? where cve_id = ? and "+ - "git_packname = ? and version = ? and cve_status in (0,1)", - status, updateTime, cveId, pakName, version).Exec() + "git_packname = ? and version = ? and cve_status = ?", + status, updateTime, cveId, pakName, version, befStatus).Exec() if err == nil { num, _ := res.RowsAffected() if num > 0 { @@ -699,11 +713,11 @@ func LockUpdateOriginStatus(updateTime, pakName, version string, cveId int64, st } } -func UnLockUpdateOriginStatus(beforeStatus, afterStatus int, updateTime string) bool { +func UnLockUpdateOriginStatus(beforeStatus, afterStatus int, updateTime, beforeTime string) bool { o := orm.NewOrm() res, err := o.Raw("UPDATE cve_origin_upstream SET "+ - "cve_status = ?, update_time = ? where cve_status = ?", - afterStatus, updateTime, beforeStatus).Exec() + "cve_status = ?, update_time = ? where cve_status = ? and update_time < ?", + afterStatus, updateTime, beforeStatus, beforeTime).Exec() if err == nil { num, _ := res.RowsAffected() if num > 0 { @@ -757,11 +771,11 @@ func LockUpdateOriginExcelStatus(updateTime string, cveId int64, status int) boo } } -func UnLockUpdateOriginExcelStatus(beforeStatus, afterStatus int, updateTime string) bool { +func UnLockUpdateOriginExcelStatus(beforeStatus, afterStatus int, updateTime, beforeTime string) bool { o := orm.NewOrm() res, err := o.Raw("UPDATE cve_origin_excel SET "+ - "cve_status = ?, update_time = ? where cve_status = ?", - afterStatus, updateTime, beforeStatus).Exec() + "cve_status = ?, update_time = ? where cve_status = ? and update_time < ?", + afterStatus, updateTime, beforeStatus, beforeTime).Exec() if err == nil { num, _ := res.RowsAffected() if num > 0 { diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go index f4984c75778aa1724227e23d8eb6a673c9ce8d15..d8f69f16810a5b0dd852d3deef1e1b0c16da2d8f 100644 --- a/cve-vulner-manager/models/issue.go +++ b/cve-vulner-manager/models/issue.go @@ -78,8 +78,8 @@ func QueryIssueByPackName(packName, days string, prcnum int) ([]VulnCenter, erro func QueryIssue(days string, prcnum int) ([]VulnCenter, error) { o := orm.NewOrm() var vc []VulnCenter - num, err := o.Raw("select * from cve_vuln_center where update_time >= ? and cve_status in (?, ?) "+ - "order by cve_id asc limit ?", days, 0, 1, prcnum).QueryRows(&vc) + num, err := o.Raw("select * from cve_vuln_center where cve_status in (?, ?) "+ + "order by cve_id asc limit ?", 0, 1, prcnum).QueryRows(&vc) if err == nil && num > 0 { logs.Info("QueryIssue, cve_vuln_center, search result: ", vc) } else { @@ -130,7 +130,6 @@ func QueryIssueScore(cveId int64) (Score, error) { return sc, err } - func GetIssueTemplateByColName(it *IssueTemplate, colName ...string) error { o := orm.NewOrm() err := o.Read(it, colName...) @@ -472,10 +471,10 @@ func LockUpdateIssueStatus(cveId int64, cveNum string, status int8) (bool) { } } -func UnLockUpdateIssueStatus(beforeStatus, afterStatus int8, updateTime string) (bool) { +func UnLockUpdateIssueStatus(beforeStatus, afterStatus int8, updateTime, beforeTime string) (bool) { o := orm.NewOrm() - res, err := o.Raw("UPDATE cve_vuln_center SET cve_status = ?,update_time = ? WHERE cve_status = ?", - afterStatus, updateTime, beforeStatus).Exec() + res, err := o.Raw("UPDATE cve_vuln_center SET cve_status = ?,update_time = ? WHERE cve_status = ? and update_time < ?", + afterStatus, updateTime, beforeStatus, beforeTime).Exec() if err == nil { num, _ := res.RowsAffected() if num > 0 { diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index 998c1c2d8547487d446ef0ceef34136214242868..39ffef13596663bb20c3ae46d9a591bebfd33f97 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -921,7 +921,8 @@ type OriginUpstreamRecord struct { FirstPerTime string `orm:"size(32);column(first_per_time)" description:"cve首次披露时间"` FirstGetTime string `orm:"size(32);column(first_get_time)" description:"cve受影响后首次感知时间"` VulStatus string `orm:"size(64);column(vul_status);null" description:"漏洞状态，REJECT, DISPUTED"` - Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求;6:低可信度数据已发邮件"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化； + 2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求;6:低可信度数据已发邮件"` AffectedScope string `orm:"size(512);column(affected_scope);null" description:"影响范围推理"` Version string `orm:"size(64);column(version);index" description:"包对应的版本号"` AttackLink string `orm:"size(512);column(attack_link);null" description:"攻击链路推理"` @@ -935,6 +936,9 @@ type OriginUpstreamRecord struct { 6:版本号未匹配、包名通过以上任意方式匹配、可信度最低"` OrgData string `orm:"type(text);column(org_data);null" description:"原始数据"` CreateTime string `orm:"size(32);column(create_time)"` + SysStatus int8 `orm:"default(0);column(sys_status)" description:"0:cve新增;1:数据已变化； + 2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求;6:低可信度数据已发邮件"` + ErrDesc string `orm:"null;type(text);column(err_desc);null" description:"数据同步失败的错误描述"` } type OpenLookengSecurityReviewer struct { diff --git a/cve-vulner-manager/task/oricvecheck.go b/cve-vulner-manager/task/oricvecheck.go index 65896ffe1e6fdc5f4f13616ce7aaf8e7439ee268..ef9f8a247b603176404b91094d401b54360600c7 100644 --- a/cve-vulner-manager/task/oricvecheck.go +++ b/cve-vulner-manager/task/oricvecheck.go @@ -36,14 +36,15 @@ func CheckOriCve() error { func UnLockTable() error { defer common.Catchs() logs.Info("Execute unlock table task start...") + beforeTime := common.GetBeforeHourTime(1) // unlock center - models.UnLockUpdateIssueStatus(15, 0, common.GetCurTime()) + models.UnLockUpdateIssueStatus(15, 0, common.GetCurTime(), beforeTime) // unlock origin cve - models.UnLockUpdateCveIssueStatus(15, 0, common.GetCurTime()) + models.UnLockUpdateCveIssueStatus(15, 0, common.GetCurTime(), beforeTime) // unlock upstream - models.UnLockUpdateOriginStatus(15, 0, common.GetCurTime()) + models.UnLockUpdateOriginStatus(15, 0, common.GetCurTime(), beforeTime) // unlock excel - models.UnLockUpdateOriginExcelStatus(15, 0, common.GetCurTime()) + models.UnLockUpdateOriginExcelStatus(15, 0, common.GetCurTime(), beforeTime) logs.Info("Execute unlock table End of task...") return nil } diff --git a/cve-vulner-manager/task/synccve.go b/cve-vulner-manager/task/synccve.go index 88ffd56d9152dae31b348c1bbf65770be223263d..d70b10111a3130587b0dcaae9b00f86e447fa19b 100644 --- a/cve-vulner-manager/task/synccve.go +++ b/cve-vulner-manager/task/synccve.go @@ -13,7 +13,6 @@ func SyncCveAndIssue() error { syncErr := SyncCveData() if syncErr != nil { logs.Error("SyncCveAndIssue, SyncCveData, err: ", syncErr) - return syncErr } issueErr := SyncCreateIssue() if issueErr != nil { diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index dd71f76fff4edb64119992e5ede792a850522cc9..4e08fdbd29f1747e8fc1c46824998e2b89b86a6f 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -741,16 +741,21 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) "and will not be processed temporarily, data: ", cveData) models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "ids or cveNum is empty", 3) return false, errors.New("数据错误，暂时不处理") } BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { logs.Error("GenCveVuler, config init error:", err, cveData.CveNum) + models.UpdateOriginStatus(common.GetCurTime(), + cveData.PackName, cveData.Version, cveData.CveId, 0) return false, err } years, confOk := BConfig.Int("cve::cve_number_t") if confOk != nil { logs.Error("GenCveVuler, config cve::cve_number_t, error:", confOk, cveData.CveNum) + models.UpdateOriginStatus(common.GetCurTime(), + cveData.PackName, cveData.Version, cveData.CveId, 0) return false, errors.New("数据错误，暂时不处理") } // Import cve as data after 2018 @@ -762,6 +767,7 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 5) logs.Error("GenCveVuler, cve: ", cveData.CveNum, ",Need to be greater than: ", years, ", Otherwise there is no need to submit an issue on git, cveData: ", cveData) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, cveData.CveNum+" cve year limited", 5) return false, errors.New("数据错误，暂时不处理") } } @@ -771,12 +777,14 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) logs.Error("GenCveVuler, QueryCveDesc, The current data description is empty "+ "and will not be processed temporarily, data: ", cveData) models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve description information is empty", 3) return false, errors.New("数据错误，暂时不处理") } cveImpact, ok := models.QueryCveImpact(cveData.CveId) if !ok { logs.Error("GenCveVuler, QueryCveImpact, Data query failed, data: ", cveData) models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 1", 3) return false, errors.New("数据错误，暂时不处理") } scopeType := "v3" @@ -797,6 +805,7 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) if scopeType == "v2" { logs.Error("GenCveVuler, QueryCveScore3, data: ", cveData, ",cveImpact: ", cveImpact) models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 2", 3) return false, errors.New("数据错误，暂时不处理") } } @@ -805,6 +814,7 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) if scopeType == "v2" { logs.Error("GenCveVuler, QueryCveCvssV2, data: ", cveData, ",cveScore: ", cveScore) models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 3", 3) return false, errors.New("数据错误，暂时不处理") } } @@ -817,7 +827,15 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) pkList := strings.Split(pk, "==") if len(pkList) == 2 { if _, ok := packNameMap[pkList[0]]; ok { - if !strings.Contains(packNameMap[pkList[0]], pkList[1]) { + versionList := strings.Split(packNameMap[pkList[0]], ",") + verExist := false + for _, vl := range versionList { + if vl == pkList[1] { + verExist = true + break + } + } + if !verExist { packNameMap[pkList[0]] += "," + pkList[1] } } else { @@ -871,6 +889,8 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 4) logs.Error("GenCveVuler, The version information corresponds to the error, ", cveData.PackName, cveData.Version, cveData.CveNum) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, + "The cve component or version information cannot be matched to the corresponding data", 4) return false, errors.New("数据错误，暂时不处理") } organizationList := make([]int8, 0) @@ -899,11 +919,15 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) } if failFlag { logs.Error("Part of the data failed") + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, + "Version does not match, data synchronization failed", 3) return false, errors.New("Part of the data failed") } } } models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 2) + UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "success", 2) return true, nil } @@ -912,38 +936,58 @@ func AddOrSelectToCenter(packageName, cveRef, scopeType, value string, cveData m cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, cveScV2 models.OriginUpstreamImpactScoreV2, organizationID int8) (bool, error) { if organizationID == 3 { - ms := models.MindSporeYaml{PackageName: packageName, Version: value} - msy, mErr := models.GetMindSporeYamlAll(&ms) - if len(msy) > 0 { - for _, my := range msy { - ok, dErr := AddOrDataToCenter(my.Repo, packageName, cveRef, scopeType, value, cveData, - openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, - cveDesc, cveScV3, goe, cveScV2, organizationID) - if !ok { - logs.Error("MindSpore, dErr: ", dErr) - return ok, dErr + if len(mindSporeVersion) > 0 { + mindMap := make(map[string]string) + for _, ver := range mindSporeVersion { + ms := models.MindSporeYaml{PackageName: packageName, Version: ver} + msy, mErr := models.GetMindSporeYamlAll(&ms) + if len(msy) > 0 { + for _, my := range msy { + if _, ok := mindMap[my.Repo]; !ok { + mindMap[my.Repo] = packageName + } + } + } else { + logs.Error("MindSpore, mErr: ", mErr) + } + } + if len(mindMap) > 0 { + for repo, _ := range mindMap { + ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, + openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) + } } } - } else { - logs.Error("MindSpore, mErr: ", mErr) - return false, mErr } } else if organizationID == 2 { - opy := models.OpenGussYaml{PackageName: packageName, Version: value} - ogy, oErr := models.GetOpengaussYamlAll(&opy) - if len(ogy) > 0 { - for _, gy := range ogy { - ok, dErr := AddOrDataToCenter(gy.Repo, packageName, cveRef, scopeType, value, cveData, - openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, - cveDesc, cveScV3, goe, cveScV2, organizationID) - if !ok { - logs.Error("openGauss, dErr: ", dErr) - return ok, dErr + if len(openGausVersion) > 0 { + gaussMap := make(map[string]string) + for _, ver := range openGausVersion { + opy := models.OpenGussYaml{PackageName: packageName, Version: ver} + ogy, oErr := models.GetOpengaussYamlAll(&opy) + if len(ogy) > 0 { + for _, gy := range ogy { + if _, ok := gaussMap[gy.Repo]; !ok { + gaussMap[gy.Repo] = packageName + } + } + } else { + logs.Error("openGauss, dErr: ", oErr) + } + } + if len(gaussMap) > 0 { + for repo, _ := range gaussMap { + ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, + openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("openGauss, dErr: ", dErr) + } } } - } else { - logs.Error("openGauss, oErr: ", oErr) - return false, oErr } } else { ok, dErr := AddOrDataToCenter(packageName, packageName, cveRef, scopeType, value, cveData, @@ -951,7 +995,7 @@ func AddOrSelectToCenter(packageName, cveRef, scopeType, value string, cveData m cveDesc, cveScV3, goe, cveScV2, organizationID) if !ok { logs.Error("openEuler, dErr: ", dErr) - return ok, dErr + return false, dErr } } return true, nil @@ -961,7 +1005,7 @@ func AddOrDataToCenter(repoNme, packageName, cveRef, scopeType, value string, cv openGausVersion, openEulerVersion, mindSporeVersion, pkList []string, openeulerNum int, cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, cveScV2 models.OriginUpstreamImpactScoreV2, organizationID int8) (bool, error) { - CveRes, err := models.QueryCveByNum(cveData.CveNum, repoNme, value, organizationID) + CveRes, err := models.QueryCveByNum(cveData.CveNum, repoNme, organizationID) if err { CveRes.RepoName = packageName retVersion := AddCveVersion(organizationID, openGausVersion, @@ -973,7 +1017,7 @@ func AddOrDataToCenter(repoNme, packageName, cveRef, scopeType, value string, cv lockx.Unlock() if !ok { logs.Error("GenCveVuler, UpdateCveGroups, cveData: ", cveData, ", err: ", err) - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + //models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) return false, errors.New("数据错误，暂时不处理") } } else { @@ -986,45 +1030,89 @@ func AddOrDataToCenter(repoNme, packageName, cveRef, scopeType, value string, cv lockx.Unlock() if !ok { logs.Error("GenCveVuler, InsertCveGroups, cveData: ", cveData, ", err: ", err) - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + //models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) return false, errors.New("数据错误，暂时不处理") } } return true, nil } +func UpdateOriginUpstreamRecord(ids, cveNum, errDesc string, errStatus int8) { + our := models.OriginUpstreamRecord{} + err := models.QueryOriginUpstreamRecord(&our, ids, cveNum) + if our.CveId > 0 { + our.SysStatus = errStatus + our.ErrDesc = errDesc + upErr := models.UpdateOriginUpstreamRecord(&our, "SysStatus", "ErrDesc") + if upErr != nil { + logs.Error("upErr: ", upErr) + } + } else { + logs.Error("err: ", err) + } +} + func AddCveVersion(organizationID int8, openGausVersion, openEulerVersion, mindSporeVersion []string, cveVersion string) (retVersion string) { retVersion = cveVersion + cveVerList := strings.Split(cveVersion, ",") if organizationID == 3 { - for _, versx := range mindSporeVersion { - if retVersion != "" && len(retVersion) > 0 { - if !strings.Contains(retVersion, versx) { + if len(cveVerList) > 0 && len(mindSporeVersion) > 0 { + for _, versx := range mindSporeVersion { + verExist := false + for _, cvl := range cveVerList { + if versx == cvl { + verExist = true + break + } + } + if !verExist { retVersion = retVersion + "," + versx } - } else { - retVersion = versx } + } else if len(mindSporeVersion) > 0 { + retVersion = strings.Join(mindSporeVersion, ",") } } else if organizationID == 2 { - for _, versx := range openGausVersion { - if retVersion != "" && len(retVersion) > 0 { - if !strings.Contains(retVersion, versx) { + if len(cveVerList) > 0 && len(openGausVersion) > 0 { + for _, versx := range openGausVersion { + verExist := false + for _, cvl := range cveVerList { + if versx == cvl { + verExist = true + break + } + } + if !verExist { retVersion = retVersion + "," + versx } - } else { - retVersion = versx } + } else if len(openGausVersion) > 0 { + retVersion = strings.Join(mindSporeVersion, ",") } } else { - for _, vers := range openEulerVersion { - versx := common.BranchVersionRep(vers) - if retVersion != "" && len(retVersion) > 0 { - if !strings.Contains(retVersion, versx) { + if len(cveVerList) > 0 && len(openEulerVersion) > 0 { + for _, vers := range openEulerVersion { + versx := common.BranchVersionRep(vers) + verExist := false + for _, cvl := range cveVerList { + if versx == cvl { + verExist = true + break + } + } + if !verExist { + retVersion = retVersion + "," + versx + } + } + } else if len(openEulerVersion) > 0 { + for i, vers := range openEulerVersion { + versx := common.BranchVersionRep(vers) + if i == 0 { + retVersion = versx + } else { retVersion = retVersion + "," + versx } - } else { - retVersion = versx } } } @@ -1144,7 +1232,7 @@ func AddExcelToCenter(cveData models.OriginExcel, cveRef string, openeulerNum in func AddExDataToCenter(cveData models.OriginExcel, cveRef, repoName string, openeulerNum int, goe models.GitPackageInfo, organizationID int8) (bool, error) { - CveRes, err := models.QueryCveByNum(cveData.CveNum, repoName, cveData.CveVersion, organizationID) + CveRes, err := models.QueryCveByNum(cveData.CveNum, repoName, organizationID) if err { CveRes.RepoName = cveData.PackName retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, @@ -1191,7 +1279,7 @@ func GetCveOriginData(prcnum, days, openeulernum, credibilityLevel int, cveRef s count = count + 1 // Add mutex lockOk := models.LockUpdateOriginStatus(common.GetCurTime(), cveOrg.PackName, - cveOrg.Version, cveOrg.CveId, 15) + cveOrg.Version, cveOrg.CveId, cveOrg.Status, 15) if !lockOk { logs.Error("Current data is being processed: PackName: ", cveOrg.PackName, cveOrg.Version, cveOrg.CveId) @@ -1931,7 +2019,7 @@ func UpdateIssueState(statusName, issueState string, issueTemp *models.IssueTemp issueTemp.MtAuditFlag = 1 issueTemp.SaAuditFlag = 1 } - if flag == 3{ + if flag == 3 { issueTemp.IssueLabel = beego.AppConfig.String("labelFixed") } } else if strings.ToLower(statusName) == "rejected" || statusName == "已拒绝" || @@ -1943,13 +2031,13 @@ func UpdateIssueState(statusName, issueState string, issueTemp *models.IssueTemp issueTemp.MtAuditFlag = 1 issueTemp.SaAuditFlag = 1 } - if flag == 3{ + if flag == 3 { issueTemp.IssueLabel = beego.AppConfig.String("labeUnaffected") } } else if strings.ToLower(statusName) == "suspended" || statusName == "已挂起" || issueState == "已挂起" { issueTemp.Status = 5 issueTemp.IssueStatus = 6 - if flag == 2 || flag == 3{ + if flag == 2 || flag == 3 { issueTemp.IssueLabel = beego.AppConfig.String("labeUnaffected") } } else { @@ -2051,7 +2139,7 @@ func GenCveVulerByIssue(cveData models.GiteOriginIssue, cveRef string, openeuler if cveData.State != "" && (strings.ToLower(cveData.State) == "rejected" || cveData.State == "已拒绝" || cveData.IssueState == "已拒绝" || cveData.IssueState == "已挂起") { logs.Info("Process data whose issue status is rejected：", cveData) - cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, hole.Version, organizationID) + cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, organizationID) if ok && cvd.CveNum != "" { lock.Lock() upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, cvd, owner) @@ -2071,7 +2159,7 @@ func GenCveVulerByIssue(cveData models.GiteOriginIssue, cveRef string, openeuler return true, nil } else if cveData.State != "" && (strings.ToLower(cveData.State) == "closed" || cveData.State == "已完成") { logs.Info("Process data whose issue status is closed：", cveData) - cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, hole.Version, organizationID) + cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, organizationID) if ok && cvd.CveNum != "" { lock.Lock() upOk, upError := UpdateIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, cvd, owner) @@ -2091,7 +2179,7 @@ func GenCveVulerByIssue(cveData models.GiteOriginIssue, cveRef string, openeuler return true, nil } else { logs.Info("Process data whose issue status is open and processing：", cveData) - cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, hole.Version, organizationID) + cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, organizationID) if ok && len(cvd.CveNum) > 1 { if openFlag == 1 { lock.Lock()