diff --git a/cve-vulner-manager/common/common.go b/cve-vulner-manager/common/common.go index 8a6631759d7872471432dfa02f9c560e577890bf..2d99ff860952fb21fd11b3461c08626b1bb6cc18 100644 --- a/cve-vulner-manager/common/common.go +++ b/cve-vulner-manager/common/common.go @@ -350,7 +350,7 @@ type CveOriginData struct { type UploadData struct { Token string `json:"Token"` - Source int `json:"source"` + Source int `json:"source"` CveData []CveOriginData } @@ -535,7 +535,7 @@ func IsContainSpecialChar(str string, flag int8) bool { } // Obtain token data from different communities and extract them into public data -func GetOwnerAndToken(cve string, organizationID int8) (owner, accessToken string) { +func GetOwnerAndToken(cve string, organizationID int8, flag ...int64) (owner, accessToken string) { if organizationID == 2 { owner = beego.AppConfig.String("opengauss::gauss_owner") accessToken = beego.AppConfig.String("opengauss::git_gauss_token") @@ -549,17 +549,19 @@ func GetOwnerAndToken(cve string, organizationID int8) (owner, accessToken strin owner = beego.AppConfig.String("gitee::owner") accessToken = beego.AppConfig.String("gitee::git_token") } - if organizationID == 1 && GetCveSource(cve) == 1 { - accessToken = beego.AppConfig.String("majun::token") + if len(flag) > 0 { + if organizationID == 1 && (GetCveSource(cve) == 1 || GetCveSourceRecord(cve) == 1) { + accessToken = beego.AppConfig.String("majun::token") + } } return } type Source struct { - Source int `orm:"column(source)"` + Source int `orm:"column(source)"` } -func GetCveSource(cve string) (source int) { +func GetCveSource(cve string) (source int) { source = 0 if len(cve) == 0 { return @@ -575,6 +577,22 @@ func GetCveSource(cve string) (source int) { return } +func GetCveSourceRecord(cve string) (source int) { + source = 0 + if len(cve) == 0 { + return + } + o := orm.NewOrm() + var res Source + err := o.Raw("select source from cve_origin_upstream_record where cve_num = ? ORDER BY cve_record_id DESC", cve).QueryRow(&res) + if err != nil { + logs.Error(err) + return + } + source = res.Source + return +} + func DeleteSliceValue(org []string, value string) (repSlice []string) { repSlice = org for k, v := range org { diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 6638d19c3c35aecccd0bd923b8f76e961b2b6410..774cd43d285efbb0051f2d3b799aba9bf7b6329b 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -172,7 +172,7 @@ cve_number_t = 2018 # Create an issue's repo whitelist;1: open; 2: close issue_whitelist = 2 # List of affected branches: openEuler-20.03-LTS,openEuler-20.03-LTS-SP1 -affected_branchs = "openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP3,openEuler-22.03-LTS" +affected_branchs = "openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP3,openEuler-22.03-LTS,openEuler-22.09" abandoned_branchs = "openEuler-20.03-LTS,openEuler-21.03,openEuler-21.09,openEuler-20.09" # Close the highest privilege of issue:1:open;2:close close_issue_privilege = 2 @@ -189,7 +189,7 @@ credibility_level = 6 # Date before adding the security bulletin link task sec_link_date = -100 # Branch configuration of cve that needs to submit an issue -submit_issue_branch = "openEuler-22.03-LTS,openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-SP3,openEuler-20.03-LTS-Next,openEuler-21.03,master" +submit_issue_branch = "openEuler-22.03-LTS,openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-SP3,openEuler-20.03-LTS-Next,openEuler-21.03,master,openEuler-22.09" # De-duplication de_duplication_date = -100 # table num diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index 8f3d9de6b238c7d4a0959a4dc40d4c4505dfb8c8..7f245d0d9f7051ac995f87f5f1d21174289002bb 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -76,8 +76,8 @@ const ( // Review reminder CommentReviewRemindMaintainer = "@%v 经过 cve-manager 解析 openEuler评分 已改变 需要等待安全组成员审核通过以后, 才能进行后续操作." CIssueType = "CVE和安全问题" - HasCreateIssue = "@%v 当前CVE:%v 已经创建过对应的ISSUE, 请不要重复创建ISSUE, cve-manager 只会识别之前的ISSUE,不会对当前ISSUE进行处理." - CreateIssueReject = "@%v 当前CVE:%v 已经创建过对应的ISSUE, 请不要重复创建ISSUE, cve-manager 只会识别之前的ISSUE,不会对当前ISSUE进行处理,当前ISSUE稍后会被工具自动设置为已拒绝." + HasCreateIssue = "@%v %v 在当前软件仓下已经创建过对应的[ISSUE](%s), 请不要重复创建" + CreateIssueReject = "@%v %v 在当前软件仓下已经创建过对应的[ISSUE](%s), 请不要重复创建, 当前ISSUE将被工具设置为已拒绝." CommentCheckVersion = "@%v 请确认分支: %v 受影响/不受影响." // rejected or upend CommentRejectedState = `@%v 当前issue状态为: %v,请先修改issue状态, 否则评论无法被识别.` @@ -3134,15 +3134,15 @@ func AddGitIssue(issueHook *models.IssuePayload, desc, product string) error { for _, cve := range cveList { item.CveNumber = cve // Check whether the current cve has created an issue - checkRes := taskhandler.OPenCheckWhetherIssue(item.CveNumber, repoPath, owner, accessToken, organizationID) + checkRes, issueNum := taskhandler.OPenCheckWhetherIssue(item.CveNumber, repoPath, owner, accessToken, organizationID) if checkRes { if organizationID == 1 { - taskhandler.AddCommentToIssue(fmt.Sprintf(CreateIssueReject, issueHook.Issue.User.UserName, item.CveNumber), + taskhandler.AddCommentToIssue(fmt.Sprintf(CreateIssueReject, issueHook.Issue.User.UserName, item.CveNumber, issueNum), issueNumber, owner, repoPath, accessToken) authToken := tokenMap[organizationID] _ = setReject(authToken.EnId, issueHook.Issue.Id, authToken.AccessToken) } else { - taskhandler.AddCommentToIssue(fmt.Sprintf(HasCreateIssue, issueHook.Issue.User.UserName, item.CveNumber), + taskhandler.AddCommentToIssue(fmt.Sprintf(HasCreateIssue, issueHook.Issue.User.UserName, item.CveNumber, issueNum), issueNumber, owner, repoPath, accessToken) } logs.Error("Cve has created an issue, please process the previous issue first, ", diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index 6f8941515f3186a5ef1c802c242a0d34677a1893..7cfe010d78d46e0286f313505b430d8cc2f00202 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -54,7 +54,7 @@ type VulnCenter struct { CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` CveLevel string `orm:"size(32);column(cve_level)" description:"nvd 致命(Critical);严重(High);中等(Medium);一般(Low);其他"` - Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求); + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求); 5:issue已经创建过，不符合要求，不处理;6:字段为空(评分,描述等);7:issue已被删除不再创建;8:分支和版本信息不想符合"` CveVersion string `orm:"size(512);column(cve_version)" description:"cve归属版本,版本合并"` RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` @@ -64,7 +64,7 @@ type VulnCenter struct { RepoName string `orm:"size(512);column(repo_name)" description:"cve对应得包名称的组件名称"` CveUrl string `orm:"type(text);column(cve_url)" description:"cve下载链接"` IsExport int8 `orm:"default(0);column(is_export)" description:"0:数据初始化;1:数据已同步到官网；2：数据不需要同步到官网；3：issue已关闭需要同步到官网"` - DataSource int8 `orm:"default(1);column(data_source)" description:"1:软件所提供;2:网络爬取；3:人工处理; 4:兼容之前的数据"` + DataSource int8 `orm:"default(1);column(data_source)" description:"1:软件所提供;2:网络爬取；3:人工处理; 4:兼容之前的数据; 5:majun提供"` CveDetailUrl string `orm:"type(text);column(cve_detail_url)" description:"cve详情链接"` OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore;4:来源openLooKeng"` CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` @@ -336,14 +336,14 @@ type OriginUpstream struct { Version string `orm:"size(64);column(version);index" description:"包对应的版本号"` AttackLink string `orm:"size(512);column(attack_link);null" description:"攻击链路推理"` IsExit int8 `orm:"default(1);column(is_exit)" description:"1: 当前包对应在src-openEuler有对应仓库; 0: 无;2:临时值"` - Credibility int `orm:"default(0);column(credibility_level)" description:"0:包名、版本号都是漏洞库原始数据; + Credibility int `orm:"default(0);column(credibility_level)" description:"0:包名、版本号都是漏洞库原始数据; 1:包名通过别名匹配、版本号为漏洞库原始数据; 2:包名版本号都通过漏洞描述获取; 3:包名通过漏洞描述获取并通过别名匹配、版本号为漏洞描述获取; 4:包名版本号通过SA获取、版本号为修复版本号、修复版本以下都视为受影响版本; 5:包名版本号通过SA获取并通过别名匹配、版本号为修复版本号、修复版本以下都视为受影响版本; 6:版本号未匹配、包名通过以上任意方式匹配、可信度最低"` - Source int `orm:"default(0);column(source)" description:"漏洞来源，0: Vtopia; 1: Majun"` + Source int `orm:"default(0);column(source)" description:"漏洞来源，0: Vtopia; 1: Majun"` CreateTime string `orm:"size(32);column(create_time)"` UpdateTime string `orm:"size(32);column(update_time);null"` DeleteTime string `orm:"size(32);column(delete_time);null"` @@ -955,25 +955,25 @@ type OriginUpstreamRecord struct { FirstPerTime string `orm:"size(32);column(first_per_time)" description:"cve首次披露时间"` FirstGetTime string `orm:"size(32);column(first_get_time)" description:"cve受影响后首次感知时间"` VulStatus string `orm:"size(64);column(vul_status);null" description:"漏洞状态，REJECT, DISPUTED"` - Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化； + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化； 2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求;6:低可信度数据已发邮件"` AffectedScope string `orm:"size(512);column(affected_scope);null" description:"影响范围推理"` Version string `orm:"size(64);column(version);index" description:"包对应的版本号"` AttackLink string `orm:"size(512);column(attack_link);null" description:"攻击链路推理"` IsExit int8 `orm:"default(1);column(is_exit)" description:"1: 当前包对应在src-openEuler有对应仓库; 0: 无;2:临时值"` - Credibility int `orm:"default(0);column(credibility_level)" description:"0:包名、版本号都是漏洞库原始数据; + Credibility int `orm:"default(0);column(credibility_level)" description:"0:包名、版本号都是漏洞库原始数据; 1:包名通过别名匹配、版本号为漏洞库原始数据; 2:包名版本号都通过漏洞描述获取; 3:包名通过漏洞描述获取并通过别名匹配、版本号为漏洞描述获取; 4:包名版本号通过SA获取、版本号为修复版本号、修复版本以下都视为受影响版本; 5:包名版本号通过SA获取并通过别名匹配、版本号为修复版本号、修复版本以下都视为受影响版本; 6:版本号未匹配、包名通过以上任意方式匹配、可信度最低"` - OrgData string `orm:"type(text);column(org_data);null" description:"原始数据"` - CreateTime string `orm:"size(32);column(create_time)"` - SysStatus int8 `orm:"default(0);column(sys_status)" description:"0:cve新增;1:数据已变化； + OrgData string `orm:"type(text);column(org_data);null" description:"原始数据"` + CreateTime string `orm:"size(32);column(create_time)"` + SysStatus int8 `orm:"default(0);column(sys_status)" description:"0:cve新增;1:数据已变化； 2：数据已处理；3：错误数据；4：版本信息错误；5：cve年份不符合要求;6:低可信度数据已发邮件"` - ErrDesc string `orm:"null;type(text);column(err_desc);null" description:"数据同步失败的错误描述"` - Source int `orm:"default(0);column(source)" description:"漏洞来源，0: Vtopia; 1: Majun"` + ErrDesc string `orm:"null;type(text);column(err_desc);null" description:"数据同步失败的错误描述"` + Source int `orm:"default(0);column(source)" description:"漏洞来源，0: Vtopia; 1: Majun"` } type OpenLookengSecurityReviewer struct { diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go index c6b7d78e0562a3931e45d0f2ae80d6c9782ad69d..27d55f28898be386119966872730275628f9c50a 100644 --- a/cve-vulner-manager/task/issuetask.go +++ b/cve-vulner-manager/task/issuetask.go @@ -152,8 +152,8 @@ compareRes: return branchFlag } -func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, flag int,list []string) error { - for ; ; { +func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, flag int, list []string) error { + for { cveData, err := models.QueryIssue(beforeTime, prcnum, list) if err == nil && len(cveData) > 0 { logs.Info("cveData: ", cveData) @@ -248,7 +248,7 @@ func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, fl if cveNumList != nil && len(cveNumList) > 1 { cveYears, yearErr := strconv.Atoi(cveNumList[1]) if yearErr == nil { - if issueValue.DataSource == 1 { + if issueValue.DataSource == 1 || issueValue.DataSource == 5 { years = toolYears } else if issueValue.DataSource == 3 { years = manYears @@ -311,7 +311,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i logs.Error("addLimitedIssue, Removed from whitelist, repo: ", issues.PackageName) continue } - for ; ; { + for { cveData, err := models.QueryIssueByPackName(issues.PackageName, beforeTime, prcnum) if err == nil && len(cveData) > 0 { logs.Info(cveData) @@ -386,7 +386,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i if cveNumList != nil && len(cveNumList) > 1 { cveYears, yearErr := strconv.Atoi(cveNumList[1]) if yearErr == nil { - if issueValue.DataSource == 1 { + if issueValue.DataSource == 1 || issueValue.DataSource == 5 { years = toolYears } else if issueValue.DataSource == 3 { years = manYears @@ -598,9 +598,9 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) er if it.Status < 3 { labelUnFix := beego.AppConfig.String("labelUnFix") labelUnFixList := taskhandler.AddLabelValue(accessToken, path, it.IssueNum, owner, labelUnFix, 1) - _,total := taskhandler.QueryIssueLabels(accessToken, path, it.IssueNum, owner) + _, total := taskhandler.QueryIssueLabels(accessToken, path, it.IssueNum, owner) if len(labelUnFixList) > 0 { - if !eqLabel(labelUnFixList,total){ + if !eqLabel(labelUnFixList, total) { it.IssueLabel = strings.Join(labelUnFixList, ",") updateBool := taskhandler.UpdateIssueLabels(accessToken, path, it.IssueNum, owner, it.IssueLabel) if !updateBool { @@ -615,7 +615,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) er return nil } -func eqLabel(a,b []string) bool { +func eqLabel(a, b []string) bool { sort.Strings(a) sort.Strings(b) @@ -772,6 +772,7 @@ func OpenLooKengIssue(issueValue models.VulnCenter, accessToken, owner, path, as //ProcIssue Deal with issue func ProcIssue(issueValue models.VulnCenter, accessToken, owner string) error { + owner, accessToken = common.GetOwnerAndToken(issueValue.CveNum, issueValue.OrganizationID, 1) assignee := "" path := issueValue.PackName // According to the requirements, the missing cve of nvd needs to submit an issue diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index baa2a7d6829922262c6524fc4889ba3393b07447..ffaaed7a1f35e2847443089eae40b60f8b516cce 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -138,12 +138,12 @@ func GetRepoMember(accessToken, owner, path string) (error, []string, []string) return errors.New("error"), assignLoginList, assignNameList } -func OPenCheckWhetherIssue(cveNumber, repoPath, owner, accessToken string, organizationID int8) bool { +func OPenCheckWhetherIssue(cveNumber, repoPath, owner, accessToken string, organizationID int8) (bool, string) { vc := models.VulnCenter{CveNum: cveNumber, PackName: repoPath, OrganizationID: organizationID} vcErr := models.GetVulnCenterByCid(&vc, "CveNum", "PackName", "OrganizationID") if vcErr != nil { logs.Error("GetVulnCenterByCid, vcErr: ", vcErr, ",cveNum: ", cveNumber) - return false + return false, "" } issueTmp := models.IssueTemplate{} issueTmp.CveNum = cveNumber @@ -154,10 +154,10 @@ func OPenCheckWhetherIssue(cveNumber, repoPath, owner, accessToken string, organ issueErr, issueBody := GetGiteeIssue(accessToken, owner, repoPath, issueTmp.IssueNum) if issueErr == nil { logs.Info("OPenCheckWhetherIssue, Issue has been created,issueBody:", issueBody) - return true + return true, issueTmp.IssueNum } } - return false + return false, "" } func CreateIssueToGit(accessToken, owner, path, assignee string, @@ -330,9 +330,9 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, labels := beego.AppConfig.String("labelUnFix") if accessToken != "" && owner != "" && path != "" { if models.FilterOldData(cve.CveNum) { - return "",nil + return "", nil } - if models.QueryCveOrigin(cve.CveNum, "update"){ + if models.QueryCveOrigin(cve.CveNum, "update") { logs.Info("update cve is not create issue , cveNum :", cve.CveNum) return "", nil } diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index 6127bac87a0c6d12095b5e2a122f8c736765c464..fa4e5fb3b22fece7a0e199645b9c84bed6930eaa 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -636,7 +636,11 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef, repoNme string, vul.PackName = repoNme vul.CveUrl = cveRef + cveData.CveNum vul.CveLevel = cveScV3.CveLevel - vul.DataSource = 1 + if common.GetCveSource(cveData.CveNum) == 1 || common.GetCveSourceRecord(cveData.CveNum) == 1 { + vul.DataSource = 5 + } else { + vul.DataSource = 1 + } vul.IsExport = 0 vul.OrganizationID = organizationID var sec models.SecurityNotice @@ -1078,7 +1082,7 @@ func AddOrDataToCenter(repoNme, packageName, cveRef, scopeType, value string, cv CveRes, err := models.QueryCveByNum(cveData.CveNum, repoNme, organizationID) if err { CveRes.RepoName = packageName - if CveRes.DataSource == 1 { + if CveRes.DataSource == 1 || CveRes.DataSource == 5 { retVersion := " " if len(versionList) > 0 { retVersion = strings.Join(versionList, ",")