From 5a24eace3258c06e65a5bc11fec6d5fff039ed18 Mon Sep 17 00:00:00 2001 From: zjm <1076115376@qq.com> Date: Wed, 1 Mar 2023 15:45:15 +0800 Subject: [PATCH] adaptation new field --- cve-vulner-manager/common/common.go | 26 ++++++++++++-- cve-vulner-manager/conf/product_app.conf | 2 +- cve-vulner-manager/controllers/cvedetail.go | 36 +++++++++++++++++++ cve-vulner-manager/controllers/upload.go | 39 ++++++++++++++++++++- cve-vulner-manager/models/cvedetail.go | 7 ++++ cve-vulner-manager/models/modeldb.go | 13 ++++++- cve-vulner-manager/models/uploadcve.go | 34 +++++++++++++++++- cve-vulner-manager/task/cve.go | 2 +- cve-vulner-manager/task/issuetask.go | 2 +- 9 files changed, 153 insertions(+), 8 deletions(-) diff --git a/cve-vulner-manager/common/common.go b/cve-vulner-manager/common/common.go index 2db1656..0ede878 100644 --- a/cve-vulner-manager/common/common.go +++ b/cve-vulner-manager/common/common.go @@ -14,8 +14,14 @@ import ( "github.com/astaxie/beego/orm" ) -const DATE_FORMAT = "2006-01-02 15:04:05" -const DATE_T_FORMAT = "2006-01-02T15:04:05" +const ( + DATE_FORMAT = "2006-01-02 15:04:05" + DATE_T_FORMAT = "2006-01-02T15:04:05" + Gemnasium = "gemnasium" + GO = "go" + GithubAdvistory = "github_advistory" + Huntr = "huntr" +) func DesString(dbpwd string) (strs string) { defer Catchs() @@ -347,8 +353,23 @@ type CveOriginData struct { Version string `json:"version"` Credibility int `json:"credibility"` Patch []CveOriginPatch `json:"patch"` + PackageUrl CvePackageUrl `json:"package_url"` +} + +type CvePackageUrl struct { + Gemnasium []PackageUrlField `json:"gemnasium"` + GO []PackageUrlField `json:"go"` + GithubAdvistory []PackageUrlField `json:"github_advistory"` + Huntr []PackageUrlField `json:"huntr"` } +type PackageUrlField struct { + Purl string `json:"purl"` + VersionStartIncluding string `json:"versionStartIncluding,omitempty"` + VersionStartExcluding string `json:"versionStartExcluding,omitempty"` + VersionEndExcluding string `json:"versionEndExcluding,omitempty"` + VersionEndIncluding string `json:"versionEndIncluding,omitempty"` +} type SbomReq struct { Coordinates []string `json:"coordinates"` } @@ -377,6 +398,7 @@ type CveOriginDetailData struct { VulType []CveVulType `json:"vulType"` FixSuggest CveFixSuggest `json:"fixSuggest"` Patch []CveOriginPatch `json:"patch"` + PackageUrl CvePackageUrl `json:"package_url"` } type CveOriginPatch struct { diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 6d5b01a..f8da0e0 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -129,7 +129,7 @@ setissueprocdateflag = 1 setissueprocdate = 0 */30 * * * * releaseUnaffectedCveflag = 1 -releaseUnaffectedCve = 0 0 11 * * 1 +releaseUnaffectedCve = 0 0 11 * * * [gitee] diff --git a/cve-vulner-manager/controllers/cvedetail.go b/cve-vulner-manager/controllers/cvedetail.go index eb5cf58..be871f9 100644 --- a/cve-vulner-manager/controllers/cvedetail.go +++ b/cve-vulner-manager/controllers/cvedetail.go @@ -297,6 +297,42 @@ func (c *CveDetailController) Get() { }) } } + + pErr, list := models.QueryOriginPackageUrl(ou.CveId) + if pErr == nil && len(list) > 0 { + p := common.CvePackageUrl{} + for _, l := range list { + switch l.Source { + case common.GO: + p.GO = append(p.GO, common.PackageUrlField{ + Purl: l.Purl, VersionStartIncluding: l.VersionStartIncluding, + VersionStartExcluding: l.VersionStartExcluding, VersionEndExcluding: l.VersionEndExcluding, + VersionEndIncluding: l.VersionEndIncluding, + }) + case common.Gemnasium: + p.Gemnasium = append(p.Gemnasium, common.PackageUrlField{ + Purl: l.Purl, VersionStartIncluding: l.VersionStartIncluding, + VersionStartExcluding: l.VersionStartExcluding, VersionEndExcluding: l.VersionEndExcluding, + VersionEndIncluding: l.VersionEndIncluding, + }) + case common.GithubAdvistory: + p.GithubAdvistory = append(p.GithubAdvistory, common.PackageUrlField{ + Purl: l.Purl, VersionStartIncluding: l.VersionStartIncluding, + VersionStartExcluding: l.VersionStartExcluding, VersionEndExcluding: l.VersionEndExcluding, + VersionEndIncluding: l.VersionEndIncluding, + }) + case common.Huntr: + p.Huntr = append(p.Huntr, common.PackageUrlField{ + Purl: l.Purl, VersionStartIncluding: l.VersionStartIncluding, + VersionStartExcluding: l.VersionStartExcluding, VersionEndExcluding: l.VersionEndExcluding, + VersionEndIncluding: l.VersionEndIncluding, + }) + + } + } + + cod.PackageUrl = p + } resp["errno"] = errcode.RecodeOk resp["errmsg"] = errcode.RecodeText(errcode.RecodeOk) resp["body"] = cod diff --git a/cve-vulner-manager/controllers/upload.go b/cve-vulner-manager/controllers/upload.go index 6ebc83f..4c3bc26 100644 --- a/cve-vulner-manager/controllers/upload.go +++ b/cve-vulner-manager/controllers/upload.go @@ -361,7 +361,44 @@ func AddOrgUpstream(source int, CveDataDict common.CveOriginData) (ResDataList [ } logs.Info("The currently inserted data already exists: ", dbCve) } - _, err := models.CreateOriginCve(CveDataDict, &orCve, &od, &ous, &osi, &osp, ose, osv, &osf) + + var packageUrls []*models.OriginUpstreamPackageUrl + + if source == 0 { + p := CveDataDict.PackageUrl + for _, i := range p.GO { + packageUrls = append(packageUrls, &models.OriginUpstreamPackageUrl{ + Purl: i.Purl, VersionStartIncluding: i.VersionStartIncluding, + VersionStartExcluding: i.VersionStartExcluding, VersionEndExcluding: i.VersionEndExcluding, + VersionEndIncluding: i.VersionEndIncluding, Source: common.GO, + }) + } + + for _, i := range p.Huntr { + packageUrls = append(packageUrls, &models.OriginUpstreamPackageUrl{ + Purl: i.Purl, VersionStartIncluding: i.VersionStartIncluding, + VersionStartExcluding: i.VersionStartExcluding, VersionEndExcluding: i.VersionEndExcluding, + VersionEndIncluding: i.VersionEndIncluding, Source: common.Huntr, + }) + } + + for _, i := range p.Gemnasium { + packageUrls = append(packageUrls, &models.OriginUpstreamPackageUrl{ + Purl: i.Purl, VersionStartIncluding: i.VersionStartIncluding, + VersionStartExcluding: i.VersionStartExcluding, VersionEndExcluding: i.VersionEndExcluding, + VersionEndIncluding: i.VersionEndIncluding, Source: common.Gemnasium, + }) + } + + for _, i := range p.GithubAdvistory { + packageUrls = append(packageUrls, &models.OriginUpstreamPackageUrl{ + Purl: i.Purl, VersionStartIncluding: i.VersionStartIncluding, + VersionStartExcluding: i.VersionStartExcluding, VersionEndExcluding: i.VersionEndExcluding, + VersionEndIncluding: i.VersionEndIncluding, Source: common.GithubAdvistory, + }) + } + } + _, err := models.CreateOriginCve(CveDataDict, &orCve, &od, &ous, &osi, &osp, ose, osv, &osf, packageUrls) if err == nil { logs.Info("Cve original data is successfully created CveNum: ", CveDataDict.Ids) ResData.CveNum = CveDataDict.Ids diff --git a/cve-vulner-manager/models/cvedetail.go b/cve-vulner-manager/models/cvedetail.go index f975a01..a31e3be 100644 --- a/cve-vulner-manager/models/cvedetail.go +++ b/cve-vulner-manager/models/cvedetail.go @@ -20,6 +20,13 @@ func GetOriginUpstream(cveNum string, ou *OriginUpstream) (err error) { return err } +func QueryOriginPackageUrl(cveid int64) (err error, list []OriginUpstreamPackageUrl) { + o := orm.NewOrm() + _, err = o.Raw("select * from cve_origin_upstream_package_url where cve_id = ?", cveid).QueryRows(&list) + + return +} + func GetOriginDesc(cveId int64, oud *OriginUpstreamDesc) (err error) { o := orm.NewOrm() err = o.Raw("select * FROM cve_origin_upstream_desc "+ diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index 30493d2..999d3be 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -356,6 +356,17 @@ type OriginUpstreamDesc struct { ZhDescription string `orm:"type(text);column(zh_desc);null" description:"cve中文描述"` } +type OriginUpstreamPackageUrl struct { + PackageId int64 `orm:"pk;auto;column(package_id)" description:"主键"` + CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` + Purl string `orm:"column(purl)" description:"purl"` + VersionStartIncluding string `orm:"column(version_start_including)" description:"最小受影响版本(包含)"` + VersionStartExcluding string `orm:"column(version_start_excluding)" description:"最小受影响版本(不包含)"` + VersionEndIncluding string `orm:"column(version_end_including)" description:"最大受影响版本(包含)"` + VersionEndExcluding string `orm:"column(version_end_excluding)" description:"最大受影响版本(不包含)"` + Source string `orm:"column(source)" description:"purl 来源"` +} + type OriginUpstreamConfig struct { ConfId int64 `orm:"pk;auto;column(conf_id)"` CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` @@ -1074,7 +1085,7 @@ func CreateDb() bool { new(MindSporeBrandTags), new(OriginUpstreamRecord), new(OpenLookengSecurityReviewer), new(OpenLookengYaml), new(IssueCommunityStatistics), new(CommunityYamlConfig), - new(IssueDeleteRecord), new(AuthTokenInfo), new(OriginUpstreamPatch), + new(IssueDeleteRecord), new(AuthTokenInfo), new(OriginUpstreamPatch), new(OriginUpstreamPackageUrl), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/cve-vulner-manager/models/uploadcve.go b/cve-vulner-manager/models/uploadcve.go index 16d66cf..56bce47 100644 --- a/cve-vulner-manager/models/uploadcve.go +++ b/cve-vulner-manager/models/uploadcve.go @@ -214,7 +214,7 @@ func QueryCveOriginByIds(ids string) (OriginUpstream, bool) { func CreateOriginCve(CveData common.CveOriginData, ou *OriginUpstream, od *OriginUpstreamDesc, ous *OriginUpstreamConfig, osi *OriginUpstreamImpact, osp *OriginUpstreamPoc, ose []*OriginUpstreamEvent, - osv []*OriginUpstreamVulType, osf *OriginUpstreamFixSuggest) (Id int64, err error) { + osv []*OriginUpstreamVulType, osf *OriginUpstreamFixSuggest, packageUrls []*OriginUpstreamPackageUrl) (Id int64, err error) { o := orm.NewOrm() errs := o.Begin() if errs == nil { @@ -477,6 +477,22 @@ func CreateOriginCve(CveData common.CveOriginData, ou *OriginUpstream, od *Origi } } + if len(packageUrls) > 0 { + p := OriginUpstreamPackageUrl{CveId: num} + o.Delete(&p, "CveId") + + for k := range packageUrls { + packageUrls[k].CveId = num + } + + if _, err = o.InsertMulti(len(packageUrls), packageUrls); err != nil { + logs.Error("CreateOriginCve, insert cve_origin_upstream_package_url failed, "+ + "ose:", packageUrls, ", err: ", err) + o.Rollback() + return 0, err + } + } + lousr := OriginUpstreamReference{CveId: num} o.Delete(&lousr, "CveId") if CveData.ReferenceData != nil && len(CveData.ReferenceData) > 0 { @@ -893,6 +909,22 @@ func CreateOriginCve(CveData common.CveOriginData, ou *OriginUpstream, od *Origi return 0, err } } + + if len(packageUrls) > 0 { + p := OriginUpstreamPackageUrl{CveId: num} + o.Delete(&p, "CveId") + + for k := range packageUrls { + packageUrls[k].CveId = num + } + + if _, err = o.InsertMulti(len(packageUrls), packageUrls); err != nil { + logs.Error("CreateOriginCve, insert cve_origin_upstream_package_url failed, "+ + "ose:", packageUrls, ", err: ", err) + o.Rollback() + return 0, err + } + } lousfs := OriginUpstreamFixSuggest{CveId: num} osErr := o.Read(&lousfs, "CveId") if osErr == orm.ErrNoRows || osErr == orm.ErrMissPK { diff --git a/cve-vulner-manager/task/cve.go b/cve-vulner-manager/task/cve.go index 97f3158..237203d 100644 --- a/cve-vulner-manager/task/cve.go +++ b/cve-vulner-manager/task/cve.go @@ -79,7 +79,7 @@ func ParamsCveOriginData() error { } func ReleaseUnaffectedCve() error { - startTime := common.TimeStrSub(common.GetCurDate(), -30) + startTime := common.TimeStrSub(common.GetCurDate(), -90) accessToken := os.Getenv("GITEE_TOKEN") if len(accessToken) == 0 { diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go index ae7291f..235e482 100644 --- a/cve-vulner-manager/task/issuetask.go +++ b/cve-vulner-manager/task/issuetask.go @@ -179,7 +179,7 @@ func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, fl // add mutex lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) if !lockErr { - logs.Error("addUnlimitedIssue, The current cve is processing, continue to process the next data, "+ + logs.Info("addUnlimitedIssue, The current cve is processing, continue to process the next data, "+ "err: ", lockErr, ",data: ", issueValue) continue } -- Gitee