From e9d00c1bfaecd987006da9e868abdc122176497b Mon Sep 17 00:00:00 2001 From: zhangjianjun Date: Thu, 23 Dec 2021 10:03:31 +0800 Subject: [PATCH 1/2] Add openLooKeng's cve code online --- cve-vulner-manager/common/common.go | 50 +- cve-vulner-manager/conf/app.conf | 12 +- cve-vulner-manager/conf/product_app.conf | 14 +- cve-vulner-manager/controllers/hook.go | 815 +++++++++++------- .../cve-py/config_yaml/deployment.yaml | 44 + .../cve-py/config_yaml/kustomization.yaml | 17 + .../cve-py/config_yaml/namespace.yaml | 6 + .../cve-py/config_yaml/secrets.yaml | 20 + .../cve-py/config_yaml/service.yaml | 14 + .../cve-py/controller/timertaskcontroller.py | 4 +- .../cve-py/tabletask/gauss_yaml.py | 32 +- .../cve-py/tabletask/mindspore_yaml.py | 32 +- .../cve-py/tabletask/openlookeng_yaml.py | 38 +- cve-vulner-manager/models/issue.go | 6 + cve-vulner-manager/models/modeldb.go | 9 +- cve-vulner-manager/models/reviewer.go | 16 + cve-vulner-manager/models/uploadcve.go | 63 ++ cve-vulner-manager/task/issue.go | 30 + cve-vulner-manager/task/issuetask.go | 108 ++- cve-vulner-manager/taskhandler/assist.go | 86 +- cve-vulner-manager/taskhandler/common.go | 103 ++- cve-vulner-manager/taskhandler/createissue.go | 93 +- cve-vulner-manager/taskhandler/cve.go | 319 ++++--- cve-vulner-manager/taskhandler/gauss.go | 3 +- cve-vulner-manager/taskhandler/hook.go | 34 +- cve-vulner-manager/taskhandler/issue.go | 11 +- cve-vulner-manager/taskhandler/oricvecheck.go | 19 +- cve-vulner-manager/util/parsepayload.go | 1 + 28 files changed, 1378 insertions(+), 621 deletions(-) create mode 100644 cve-vulner-manager/cve-py/config_yaml/deployment.yaml create mode 100644 cve-vulner-manager/cve-py/config_yaml/kustomization.yaml create mode 100644 cve-vulner-manager/cve-py/config_yaml/namespace.yaml create mode 100644 cve-vulner-manager/cve-py/config_yaml/secrets.yaml create mode 100644 cve-vulner-manager/cve-py/config_yaml/service.yaml diff --git a/cve-vulner-manager/common/common.go b/cve-vulner-manager/common/common.go index 64de615..044ba72 100644 --- a/cve-vulner-manager/common/common.go +++ b/cve-vulner-manager/common/common.go @@ -464,10 +464,56 @@ func SliceRemoveDup(req interface{}) (ret []interface{}) { } value := reflect.ValueOf(req) for i := 0; i < value.Len(); i++ { - if i > 0 && reflect.DeepEqual(value.Index(i-1).Interface(), value.Index(i).Interface()) { + if i > 0 && reflect.DeepEqual(value.Index(i - 1).Interface(), value.Index(i).Interface()) { continue } ret = append(ret, value.Index(i).Interface()) } return -} \ No newline at end of file +} + +func IsContainsLetters(str string) bool { + if len(str) > 0 { + if isOk, _ := regexp.MatchString("[a-zA-Z-_]+", str); isOk { + return true + } + } else { + return true + } + return false +} + +func IsContainSpecialChar(str string, flag int8) bool { + if len(str) > 0 { + if flag == 1 { + if isOk, _ := regexp.MatchString("^[r|R][0-9]+(.)*[0-9]$", str); isOk { + return true + } + } else { + if isOk, _ := regexp.MatchString("^[v|V][0-9]+(.)*[0-9]$", str); isOk { + return true + } + } + } else { + return true + } + return false +} + +// Obtain token data from different communities and extract them into public data +func GetOwnerAndToken(organizationID int8) (owner, accessToken string) { + if organizationID == 2 { + owner = beego.AppConfig.String("opengauss::gauss_owner") + accessToken = beego.AppConfig.String("opengauss::git_gauss_token") + } else if organizationID == 3 { + owner = beego.AppConfig.String("mindspore::mindspore_owner") + accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") + } else if organizationID == 4 { + owner = beego.AppConfig.String("openlookeng::openlookeng_owner") + accessToken = beego.AppConfig.String("openlookeng::git_openlookeng_token") + } else { + owner = beego.AppConfig.String("gitee::owner") + accessToken = beego.AppConfig.String("gitee::git_token") + } + return +} diff --git a/cve-vulner-manager/conf/app.conf b/cve-vulner-manager/conf/app.conf index 9e8f50c..6501f27 100644 --- a/cve-vulner-manager/conf/app.conf +++ b/cve-vulner-manager/conf/app.conf @@ -32,7 +32,7 @@ gaussFileDir = "download/gauss" # release package download url http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" #communityName = "src-openeuler:1,opengauss:2,mindspore:3" -communityName = "cve-test:1,cve-gauss:2,cve-mindspore:3" +communityName = "cve-test:1,cve-gauss:2,cve-mindspore:3,open-loo-keng:4" [mysql] @@ -172,12 +172,16 @@ sec_link_date = -100 submit_issue_branch = "openEuler-20.03-LTS,openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-Next,openEuler-21.03,master" # De-duplication de_duplication_date = -300 +# table num +yaml_config_table = "cve_open_euler_repo_origin:1,cve_open_guss_yaml:2,cve_mind_spore_yaml:3,cve_open_lookeng_yaml:4" +bot_cu_account = "openeuler-ci-bot,opengauss-bot,mindspore-ci-bot,mindspore_ci,i-robot,CVE,zhangjianjun_code" [reflink] -comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual.md +comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/cve-vulner-manager/doc/md/manual.md gauss_comment_cmd = https://gitee.com/opengauss/security/blob/master/cve/manual.md spore_comment_cmd = https://gitee.com/mindspore/community/blob/master/security/cve_issue_template.md +looKeng_comment_cmd = https://gitee.com/openlookeng/community/blob/master/security/cve/doc/md/manual.md openeuler_web = https://www.openeuler.org #openeuler_web = http://119.13.93.224:9090 get_cve_link = http://127.0.0.1:8080/pulls/cve/info @@ -240,7 +244,7 @@ url = "http://cve-manager-agent.cve-manager-agent.svc.cluster.local/v1/cve/track [openlookeng] #openlookeng_owner = openlookeng -openlookeng_owner = cve-openlookeng +openlookeng_owner = open-loo-keng # git token git_openlookeng_token = "${GITEE_OPENLOOKENG_TOKEN||xxx}" -openlookeng_version = "master,r1.2" \ No newline at end of file +openlookeng_version = "master" \ No newline at end of file diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 0f42adc..e9ff8ac 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -31,7 +31,7 @@ saFileDir = "download/sa" gaussFileDir = "download/gauss" # release package download url rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" -communityName = "src-openeuler:1,opengauss:2,mindspore:3" +communityName = "src-openeuler:1,opengauss:2,mindspore:3,openlookeng:4" [mysql] @@ -164,12 +164,16 @@ sec_link_date = -100 submit_issue_branch = "openEuler-20.03-LTS,openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-Next,openEuler-21.03,master" # De-duplication de_duplication_date = -100 +# table num +yaml_config_table = "cve_open_euler_repo_origin:1,cve_open_guss_yaml:2,cve_mind_spore_yaml:3,cve_open_lookeng_yaml:4" +bot_cu_account = "openeuler-ci-bot,opengauss-bot,mindspore-ci-bot,mindspore_ci,i-robot,CVE" [reflink] -comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual.md +comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/cve-vulner-manager/doc/md/manual.md gauss_comment_cmd = https://gitee.com/opengauss/security/blob/master/cve/manual.md spore_comment_cmd = https://gitee.com/mindspore/community/blob/master/security/cve_issue_template.md +looKeng_comment_cmd = https://gitee.com/openlookeng/community/blob/master/security/cve/doc/md/manual.md openeuler_web = https://www.openeuler.org get_cve_link = http://cve-manager-analysis-service.cve-manager.svc.cluster.local:8080/pulls/cve/info @@ -201,7 +205,7 @@ email_port = 25 gauss_owner = opengauss # git token git_gauss_token = "${GITEE_GAUSS_TOKEN||xxx}" -gauss_version = "2.0.0,master" +gauss_version = "master" gauss_issue_path = security gauss_branch_path = openGauss-server sa_init_value = 1001 @@ -213,7 +217,7 @@ nvd_relink = https://nvd.nist.gov/vuln/detail/ mindspore_owner = mindspore # git token git_mindspore_token = "${GITEE_MINDSPORE_TOKEN||xxx}" -mindspore_version = "master,r1.2" +mindspore_version = "master" [obs] access_key_id = "${OBS_KEY_ID||xxx}" @@ -231,4 +235,4 @@ url = "http://cve-manager-agent.cve-manager-agent.svc.cluster.local/v1/cve/track openlookeng_owner = openlookeng # git token git_openlookeng_token = "${GITEE_OPENLOOKENG_TOKEN||xxx}" -openlookeng_version = "master,r1.2" \ No newline at end of file +openlookeng_version = "master" \ No newline at end of file diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index c89340e..ecbb636 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "regexp" + "sort" "strconv" "strings" "sync" @@ -80,6 +81,7 @@ const ( // Get cve information comment CommentGetNvdCveSuccess = `@%v CVE信息同步成功, 稍后请重新加载页面.` CommentGetNvdCveFailed = `@%v CVE信息同步失败, 请稍后重试, 或者数据源不存在.` + CommentRepeatIssue = `%v 请检查当前: %v,是否重复创建, issue编号： %v, 重复创建的issue,将不会被再次识别.` ) var comLock sync.Mutex @@ -170,18 +172,21 @@ func (c *HookEventControllers) handleIssue() { return } cuAccount := issueHook.Sender.Login - if cuAccount != "" && len(cuAccount) > 1 { - if cuAccount == "openeuler-ci-bot" { - logs.Error("openeuler-ci-bot, Ignore this comment") - return - } - if cuAccount == "opengauss-bot" { - logs.Error("opengauss-bot, Ignore this comment") - return - } - if cuAccount == "mindspore-ci-bot" || cuAccount == "mindspore_ci" || cuAccount == "i-robot" { - logs.Error("mindspore-ci-bot, Ignore this comment") - return + if issueHook.Issue.Number == "" || cuAccount == "" { + logs.Error("Data has null values: issueNum, cuAccount: ", issueHook.Issue.Number, cuAccount) + return + } + nameSpace := util.TrimString(issueHook.Repository.NameSpace) + organizationID := int8(1) + organizationID = taskhandler.GetOrganizationId(nameSpace) + botCuAccountStr := beego.AppConfig.String("cve::bot_cu_account") + botCuAccountList := strings.Split(botCuAccountStr, ",") + if len(botCuAccountList) > 0 { + for _, botCu := range botCuAccountList { + if cuAccount == botCu { + logs.Error(cuAccount, ", Ignore this comment") + return + } } } hookPwd := beego.AppConfig.String("hook::hookpwd") @@ -216,17 +221,23 @@ func (c *HookEventControllers) handleIssue() { issueTmp := models.IssueTemplate{IssueNum: issueHook.Iid, IssueId: issueHook.Issue.Id} err := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") if err == nil { - logs.Error(err) - return + vc := models.VulnCenter{CveId: issueTmp.CveId} + vcErr := models.GetVulnCenterByCid(&vc, "CveId") + if vcErr == nil && vc.OrganizationID == organizationID { + owner, token := common.GetOwnerAndToken(organizationID) + cc := fmt.Sprintf(CommentRepeatIssue, "@"+cuAccount, issueTmp.CveNum, issueTmp.IssueNum) + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) + return + } } - err = gitAddIssueProc(&issueHook) + err = gitAddIssueProc(&issueHook, organizationID) if err != nil { logs.Error(err) return } } if issueHook.Action == "delete" { - err = gitDelIssueProc(&issueHook) + err = gitDelIssueProc(&issueHook, organizationID) if err != nil { logs.Error(err) return @@ -333,7 +344,7 @@ func getMaintainer(path, prSender, assignee string) string { return maintainerVaule } -func gaussCloseIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, +func otherCloseIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, token, owner, fixed, unFix, path string, cveCenter *models.VulnCenter) { unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, issueTmp.OwnedVersion, cveCenter.OrganizationID) @@ -351,70 +362,14 @@ func gaussCloseIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueT content := fmt.Sprintf("%v 仓库的CVE和安全问题的ISSUE,CVE编号: %v,", issueTmp.Repo, issueTmp.CveNum) taskhandler.SendPrivateLetters(token, content, issueHook.Sender.UserName) } - return - } - if _, tb, ok := checkGaussIssueClosedAnalysisComplete(issueTmp); !ok { - //send comment to issue - issueTmp.IssueStatus = 1 - issueTmp.IssueLabel = unFix - issueTmp.StatusName = "open" - _, issueErr := taskhandler.UpdateIssueToGit(token, owner, path, - *cveCenter, *issueTmp) - if issueErr == nil { - na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" - cc := fmt.Sprintf(ContentReview, "@"+issueHook.Sender.UserName) + tb + na - taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) - } } else { - issueTmp.IssueLabel = unFix - issueTmp.StatusName = "open" - issueTmp.Status = 1 - assignee := "@" + issueTmp.Assignee - issuePrFlag := VerifyIssueAsPr(issueTmp, *cveCenter, false, - assignee, issueHook.Sender.UserName) - if issuePrFlag { - //1. change issue status - issueTmp.IssueStatus = 2 - //issueTmp.Status = 3 - cveCenter.IsExport = 3 - issueTmp.StatusName = issueHook.Issue.StateName - issueTmp.Status = 3 - if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { - issueTmp.IssueStatus = 2 - cveCenter.IsExport = 3 - issueTmp.IssueLabel = fixed - } else { - issueTmp.IssueStatus = 6 - cveCenter.IsExport = 2 - issueTmp.IssueLabel = unFix - } - } else { - issueTmp.IssueStatus = 1 - cveCenter.IsExport = 0 - } - } -} - -func sporeCloseIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, - token, owner, fixed, unFix, path string, cveCenter *models.VulnCenter) { - unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, - issueTmp.OwnedVersion, cveCenter.OrganizationID) - if len(unFixList) > 0 { - //send comment to issue - issueTmp.IssueStatus = 1 - issueTmp.IssueLabel = unFix - issueTmp.StatusName = "open" - _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, - *cveCenter, *issueTmp) - if issueErr == nil { - na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" - cc := fmt.Sprintf(CommentCheckVersion, issueHook.Sender.UserName, strings.Join(unFixList, ",")) + na - taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, issueTmp.Repo, token) - content := fmt.Sprintf("%v 仓库的CVE和安全问题的ISSUE,CVE编号: %v,", issueTmp.Repo, issueTmp.CveNum) - taskhandler.SendPrivateLetters(token, content, issueHook.Sender.UserName) + commonFunc := checkGaussIssueClosedAnalysisComplete + if cveCenter.OrganizationID == 4 { + commonFunc = checkLooKengIssueClosedAnalysisComplete + } else if cveCenter.OrganizationID == 3 { + commonFunc = checkSporeIssueClosedAnalysisComplete } - } else { - if _, tb, ok := checkSporeIssueClosedAnalysisComplete(issueTmp); !ok { + if _, tb, ok := commonFunc(issueTmp); !ok { //send comment to issue issueTmp.IssueStatus = 1 issueTmp.IssueLabel = unFix @@ -600,20 +555,7 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { models.UpdateIssueTemplate(&issueTmp, "Status") return errors.New("The current issue has been rejected and will not be processed") } - token := beego.AppConfig.String("gitee::git_token") - owner := beego.AppConfig.String("gitee::owner") - if cveCenter.OrganizationID == 2 { - gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") - gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") - owner = gaussOwner - token = gitGaussToken - } else if cveCenter.OrganizationID == 3 { - mindsporeOwner := beego.AppConfig.String("mindspore::mindspore_owner") - gitMindsporeToken := beego.AppConfig.String("mindspore::git_mindspore_token") - owner = mindsporeOwner - token = gitMindsporeToken - // Query the repo that needs to submit an issue - } + owner, token := common.GetOwnerAndToken(cveCenter.OrganizationID) issueTmp.StatusName = issueHook.Issue.StateName logs.Info("Initiating issue status modification, sponsor: @", issueHook.Sender.UserName, ", Modify status: ", issueHook.Issue.StateName, ", data: ", issueHook) @@ -626,6 +568,8 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { checkFunc = checkGaussIssueAnalysisComplete } else if cveCenter.OrganizationID == 3 { checkFunc = checkSporeIssueAnalysisComplete + } else if cveCenter.OrganizationID == 4 { + checkFunc = checkLooKengIssueAnalysisComplete } _, _, ok := checkFunc(&issueTmp) if ok { @@ -643,6 +587,8 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { checkFunc = checkGaussIssueAnalysisComplete } else if cveCenter.OrganizationID == 3 { checkFunc = checkSporeIssueAnalysisComplete + } else if cveCenter.OrganizationID == 4 { + checkFunc = checkLooKengIssueAnalysisComplete } _, _, ok := checkFunc(&issueTmp) if ok { @@ -657,12 +603,10 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { logs.Error("The issue has been closed and cannot be operated again,issuetmp: ", issueTmp) return errors.New("The issue has been closed and cannot be operated again") } - if cveCenter.OrganizationID == 3 { - sporeCloseIssueProc(issueHook, &issueTmp, token, owner, fixed, unFix, path, &cveCenter) - } else if cveCenter.OrganizationID == 2 { - gaussCloseIssueProc(issueHook, &issueTmp, token, owner, fixed, unFix, path, &cveCenter) - } else { + if cveCenter.OrganizationID == 1 { closeIssueProc(issueHook, &issueTmp, token, owner, fixed, unFix, &cveCenter) + } else { + otherCloseIssueProc(issueHook, &issueTmp, token, owner, fixed, unFix, path, &cveCenter) } case IssueRejectState: issueTmp.Status = 4 @@ -688,59 +632,65 @@ func VerifyIssueAsPr(issueTmp *models.IssueTemplate, cveCenter models.VulnCenter } affectBranchsxList := make([]string, 0) affectedBranchs := "" - token := "" - owner := "" path := cveCenter.PackName - if cveCenter.OrganizationID == 3 { + affectProductList := make([]string, 0) + tmpAffectBranchsxList := make([]string, 0) + owner, token := common.GetOwnerAndToken(cveCenter.OrganizationID) + if cveCenter.OrganizationID == 4 { + affectedBranchs = beego.AppConfig.String("openlookeng::openlookeng_version") + affectBranchsxList = taskhandler.CreateBrandAndTags(token, owner, path, cveCenter.OrganizationID) + } else if cveCenter.OrganizationID == 3 { affectedBranchs = beego.AppConfig.String("mindspore::mindspore_version") - owner = beego.AppConfig.String("mindspore::mindspore_owner") - token = beego.AppConfig.String("mindspore::git_mindspore_token") - // Query the repo that needs to submit an issue - if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { - tmpTagList := make([]string, 0) - affectProductSlice := strings.Split(sn.AffectProduct, "/") - for _, tags := range affectProductSlice { - mdbt := models.MindSporeBrandTags{PackageName: path, Tags: tags} - mtErr := models.QueryMindSporeBrandTags(&mdbt, "PackageName", "Tags") - if mtErr == nil { - tmpTagList = append(tmpTagList, mdbt.Brand) - } - } - //logs.Info("tmpTagList ===> ", tmpTagList) - if len(tmpTagList) > 0 { - sn.AffectProduct = strings.Join(tmpTagList, "/") - } else { - sn.AffectProduct = "" - } - } + affectBranchsxList = taskhandler.CreateBrandAndTags(token, owner, path, cveCenter.OrganizationID) } else if cveCenter.OrganizationID == 2 { - affectedBranchs = beego.AppConfig.String("opengauss::gauss_version") - token = beego.AppConfig.String("opengauss::git_gauss_token") - owner = beego.AppConfig.String("opengauss::gauss_owner") + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") if len(path) < 2 { - issuePath := beego.AppConfig.String("opengauss::gauss_issue_path") - path = issuePath + path = beego.AppConfig.String("opengauss::gauss_issue_path") } + affectBranchsxList, _ = taskhandler.GetBranchesInfo(token, owner, path, cveCenter.OrganizationID) } else { affectedBranchs = beego.AppConfig.String("cve::affected_branchs") - token = beego.AppConfig.String("gitee::git_token") - owner = beego.AppConfig.String("gitee::owner") path = issueTmp.Repo + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } } - if affectedBranchs != "" && len(affectedBranchs) > 0 { - affectBranchsxList = strings.Split(affectedBranchs, ",") - } + if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { + tmpAffectBranchsxList = strings.Split(sn.AffectProduct, "/") + } + if len(tmpAffectBranchsxList) > 0 { + sort.Strings(tmpAffectBranchsxList) + repSlice := common.SliceRemoveDup(tmpAffectBranchsxList) + for _, rep := range repSlice { + if len(rep.(string)) > 0 { + affectProductList = append(affectProductList, rep.(string)) + } + } + } + + if len(affectProductList) > 0 { issueTmp.SaAuditFlag = 0 - affectProductList := strings.Split(sn.AffectProduct, "/") branchMaps := make(map[string]bool) for _, brands := range affectProductList { brands = common.BranchVersionRep(brands) if len(affectBranchsxList) > 0 { keyBandList := []string{} for _, affectBranch := range affectBranchsxList { + affectBranch = common.BranchVersionRep(affectBranch) if affectBranch == brands { - keyBandList = append(keyBandList, affectBranch) + // Query the repo that needs to submit an issue + if cveCenter.OrganizationID == 3 { + mdbt := models.MindSporeBrandTags{PackageName: path, Tags: affectBranch} + mtErr := models.QueryMindSporeBrandTags(&mdbt, "PackageName", "Tags") + if mtErr == nil { + keyBandList = append(keyBandList, mdbt.Brand) + } + } else if cveCenter.OrganizationID == 4 { + keyBandList = append(keyBandList, "master") + } else { + keyBandList = append(keyBandList, affectBranch) + } } } if len(keyBandList) > 0 { @@ -1094,8 +1044,9 @@ func updateTempAndCenter(issueTmp models.IssueTemplate, cveCenter models.VulnCen if issueTmp.Status > 2 { affectBranchsxList := make([]string, 0) affectedBranchs := "" - if cveCenter.OrganizationID == 3 { - affectedBranchs = beego.AppConfig.String("mindspore::mindspore_version") + if cveCenter.OrganizationID == 4 { + affectedBranchs = beego.AppConfig.String("openlookeng::openlookeng_version") + } else if cveCenter.OrganizationID == 3 { if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { tmpTagList := make([]string, 0) affectProductSlice := strings.Split(sn.AffectProduct, "/") @@ -1108,12 +1059,13 @@ func updateTempAndCenter(issueTmp models.IssueTemplate, cveCenter models.VulnCen } if len(tmpTagList) > 0 { sn.AffectProduct = strings.Join(tmpTagList, "/") + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") } else { sn.AffectProduct = "" } } } else if cveCenter.OrganizationID == 2 { - affectedBranchs = beego.AppConfig.String("opengauss::gauss_version") + affectedBranchs = strings.ReplaceAll(sn.AffectProduct, "/", ",") } else { affectedBranchs = beego.AppConfig.String("cve::affected_branchs") } @@ -1320,6 +1272,64 @@ func gaussMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, to } } +func looKengMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, + unfixed, path string, cveCenter models.VulnCenter) { + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, + issueTmp.OwnedVersion, cveCenter.OrganizationID) + if len(unFixList) > 0 { + na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(CommentCheckVersion, cuAccount, strings.Join(unFixList, ",")) + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + return + } + if _, tb, ok := checkLooKengIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(AnalysisComplete, cuAccount) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + return + } else { + if !isLooKengReviewer(cuAccount) && issueTmp.Assignee != cuAccount { + logs.Error("Invalid user review, cuAccount: ", cuAccount) + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v maintainer具有通过(/approve或者/close)关闭issue, 否则请通过issue页面按钮关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, path, token) + return + } + issueTmp.IssueLabel = unfixed + issueTmp.StatusName = "open" + issueTmp.Status = 1 + assignee := "@" + issueTmp.Assignee + issuePrFlag := VerifyIssueAsPr(issueTmp, cveCenter, false, assignee, cuAccount) + if issuePrFlag { + issueTmp.IssueLabel = fixed + issueTmp.StatusName = "closed" + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v 你已审核模板内容, cve-manager 将关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, path, token) + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, *issueTmp) + if issueErr == nil { + logs.Info("Initiate an issue to close,issuetmp: ", issueTmp) + } else { + logs.Error("Issue closing operation failed, issuetmp: ", issueTmp, ",issueErr: ", issueErr) + return + } + //issueTmp.SaAuditFlag = 1 + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + } + updateBool := updateTempAndCenter(*issueTmp, cveCenter, token, owner) + if !updateBool { + return + } + } + } +} + func sporeMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, unfixed, path string, cveCenter models.VulnCenter) { unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, @@ -1528,18 +1538,14 @@ func handleIssueComment(payload models.CommentPayload) { return } // Ignore this comment - if cuAccount != "" && len(cuAccount) > 1 { - if cuAccount == "openeuler-ci-bot" { - logs.Error("openeuler-ci-bot, Ignore this comment") - return - } - if cuAccount == "opengauss-bot" { - logs.Error("opengauss-bot, Ignore this comment") - return - } - if cuAccount == "mindspore-ci-bot" || cuAccount == "mindspore_ci" || cuAccount == "i-robot" { - logs.Error("mindspore-ci-bot, Ignore this comment") - return + botCuAccountStr := beego.AppConfig.String("cve::bot_cu_account") + botCuAccountList := strings.Split(botCuAccountStr, ",") + if len(botCuAccountList) > 0 { + for _, botCu := range botCuAccountList { + if cuAccount == botCu { + logs.Error(cuAccount, ", Ignore this comment") + return + } } } issueTmp := models.IssueTemplate{IssueNum: issueNum, IssueId: issueId} @@ -1548,8 +1554,7 @@ func handleIssueComment(payload models.CommentPayload) { logs.Error(err) return } - accessToken := os.Getenv("GITEE_TOKEN") - owner := beego.AppConfig.String("gitee::owner") + fixed := beego.AppConfig.String("labelFixed") unfixed := beego.AppConfig.String("labelUnFix") path := issueTmp.Repo @@ -1559,18 +1564,17 @@ func handleIssueComment(payload models.CommentPayload) { logs.Error("GetVulnCenterByCid, vcErr: ", vcErr, ",CveId: ", issueTmp.CveId) return } + owner, accessToken := common.GetOwnerAndToken(vc.OrganizationID) if vc.OrganizationID == 2 { - owner = beego.AppConfig.String("opengauss::gauss_owner") - accessToken = beego.AppConfig.String("opengauss::git_gauss_token") if len(path) < 2 { path = beego.AppConfig.String("opengauss::gauss_issue_path") } cBody = strings.ReplaceAll(cBody, util.KwOpenGaussScore, util.KwOpenEulerScore) } else if vc.OrganizationID == 3 { - owner = beego.AppConfig.String("mindspore::mindspore_owner") - accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") // Query the repo that needs to submit an issue cBody = strings.ReplaceAll(cBody, util.KwMindSporeScore, util.KwOpenEulerScore) + } else if vc.OrganizationID == 4 { + cBody = strings.ReplaceAll(cBody, util.KwLooKengScore, util.KwOpenEulerScore) } if len(repoPath) > 1 && repoPath != path { vc.PackName = repoPath @@ -1621,7 +1625,11 @@ func handleIssueComment(payload models.CommentPayload) { logs.Error("The issue has been closed and cannot be operated again,issuetmp: ", issueTmp) return } - if vc.OrganizationID == 3 { + if vc.OrganizationID == 4 { + comLock.Lock() + looKengMaintainerApprove(&issueTmp, cuAccount, owner, accessToken, fixed, unfixed, path, vc) + comLock.Unlock() + } else if vc.OrganizationID == 3 { comLock.Lock() sporeMaintainerApprove(&issueTmp, cuAccount, owner, accessToken, fixed, unfixed, path, vc) comLock.Unlock() @@ -1766,6 +1774,11 @@ func isMindSporeReviewer(path string) bool { return sr.Read("name_space") } +func isLooKengReviewer(path string) bool { + sr := models.OpenLookengSecurityReviewer{NameSpace: path} + return sr.Read("name_space") +} + func analysisComment(owner, accessToken, path string, cuAccount string, cBody string, payload *models.CommentPayload, issueTmp models.IssueTemplate, OrganizationID int8) { if issueTmp.Status == 3 { @@ -1869,6 +1882,8 @@ func analysisComment(owner, accessToken, path string, cuAccount string, cBody st checkFunc = checkGaussIssueAnalysisComplete } else if OrganizationID == 3 { checkFunc = checkSporeIssueAnalysisComplete + } else if OrganizationID == 4 { + checkFunc = checkLooKengIssueAnalysisComplete } if msg, tb, ok := checkFunc(&issueTmp); !ok { //send comment to issue @@ -1886,25 +1901,7 @@ func analysisComment(owner, accessToken, path string, cuAccount string, cBody st msg = fmt.Sprintf(CommentAnalysisCplTpl, assignee, msg) taskhandler.AddCommentToIssue(msg, issueTmp.IssueNum, owner, path, accessToken) } else { - if OrganizationID == 3 { - na := "\n**请确认分析内容的准确性, 确认无误后, 您可以进行后续步骤, 否则您可以继续分析.**" - cc := fmt.Sprintf(AnalysisComplete, issueTmp.Assignee) + tb + na - taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) - // change score status - err := changeOpenEulerScoreStatus(issueTmp.CveId, 3) - if err != nil { - logs.Error(err) - } - } else if OrganizationID == 2 { - na := "\n**请确认分析内容的准确性, 确认无误后, 您可以进行后续步骤, 否则您可以继续分析.**" - cc := fmt.Sprintf(AnalysisComplete, issueTmp.Assignee) + tb + na - taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) - // change score status - err := changeOpenEulerScoreStatus(issueTmp.CveId, 3) - if err != nil { - logs.Error(err) - } - } else { + if OrganizationID == 1 { //1. change issue status issueTmp.IssueStatus = 3 //2. Are the cvsScore and openEuler score equal .If not equal, notify the auditor to review . @@ -1946,6 +1943,15 @@ func analysisComment(owner, accessToken, path string, cuAccount string, cBody st logs.Error(err) } } + } else { + na := "\n**请确认分析内容的准确性, 确认无误后, 您可以进行后续步骤, 否则您可以继续分析.**" + cc := fmt.Sprintf(AnalysisComplete, issueTmp.Assignee) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) + // change score status + err := changeOpenEulerScoreStatus(issueTmp.CveId, 3) + if err != nil { + logs.Error(err) + } } } err := models.UpdateIssueTemplate(&issueTmp, "issue_status", "mt_audit_flag") @@ -2009,12 +2015,12 @@ func checkIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok ok = true tbContent := make([]interface{}, 15) if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { - msg = fmt.Sprintf("影响性分析说明 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") ok = false return } tbContent[0] = "已分析" - tbContent[1] = "影响性分析说明" + tbContent[1] = "1.影响性分析说明" tbContent[2] = util.TrimStringNR(i.CveAnalysis) affectedVersionFlag := 1 if i.AffectedVersion != "" { @@ -2043,48 +2049,48 @@ func checkIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok } } if !versionfFlag { - msg = fmt.Sprintf("受影响版本排查(受影响/不受影响): 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) ok = false return } if versionfFlag { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = util.TrimStringNR(i.AffectedVersion) } else { tbContent[9] = "待分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = util.TrimStringNR(i.AffectedVersion) } } else { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = "" } if affectedVersionFlag == 1 { tbContent[3] = "已分析" - tbContent[4] = "openEulerScore" + tbContent[4] = "2.openEulerScore" tbContent[5] = i.OpenEulerScore tbContent[6] = "已分析" - tbContent[7] = "openEulerVector" + tbContent[7] = "2.openEulerVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } else { if i.OpenEulerScore == 0.0 { - msg = fmt.Sprintf("openEulerScore没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.openEulerScore=> 没有填写或正确填写(0-10)") ok = false return } tbContent[3] = "已分析" - tbContent[4] = "openEulerScore" + tbContent[4] = "2.openEulerScore" tbContent[5] = i.OpenEulerScore if i.OpenEulerVector == "" || len(i.OpenEulerVector) < 1 { - msg = fmt.Sprintf("openEulerVector没有正确填写") + msg = fmt.Sprintf("2.openEulerVector=> 没有正确填写") ok = false return } tbContent[6] = "已分析" - tbContent[7] = "openEulerVector" + tbContent[7] = "2.openEulerVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } if i.AbiVersion != "" { @@ -2108,22 +2114,22 @@ func checkIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok } } if !versionAbiFlag { - msg = fmt.Sprintf("修复是否涉及abi变化(是/否): 没有分析或未按正确格式填写:%v", i.AbiVersion) + msg = fmt.Sprintf("4.修复是否涉及abi变化(是/否)=> 没有分析或未按正确格式填写:%v", i.AbiVersion) ok = false return } if versionAbiFlag { tbContent[12] = "已分析" - tbContent[13] = "修复是否涉及abi变化" + tbContent[13] = "4.修复是否涉及abi变化" tbContent[14] = util.TrimStringNR(i.AbiVersion) } else { tbContent[12] = "待分析" - tbContent[13] = "修复是否涉及abi变化" + tbContent[13] = "4.修复是否涉及abi变化" tbContent[14] = util.TrimStringNR(i.AbiVersion) } } else { tbContent[12] = "已分析" - tbContent[13] = "修复是否涉及abi变化" + tbContent[13] = "4.修复是否涉及abi变化" tbContent[14] = "" } tbStr = fmt.Sprintf(tb, tbContent...) @@ -2146,12 +2152,12 @@ func checkGaussIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string ok = true tbContent := make([]interface{}, 12) if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { - msg = fmt.Sprintf("影响性分析说明 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") ok = false return } tbContent[0] = "已分析" - tbContent[1] = "影响性分析说明" + tbContent[1] = "1.影响性分析说明" tbContent[2] = util.TrimStringNR(i.CveAnalysis) affectedVersionFlag := 1 if i.AffectedVersion != "" { @@ -2179,48 +2185,48 @@ func checkGaussIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string } } if !versionfFlag { - msg = fmt.Sprintf("受影响版本排查(受影响/不受影响): 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) ok = false return } if versionfFlag { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = util.TrimStringNR(i.AffectedVersion) } else { tbContent[9] = "待分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = util.TrimStringNR(i.AffectedVersion) } } else { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = "" } if affectedVersionFlag == 1 { tbContent[3] = "已分析" - tbContent[4] = "openGaussScore" + tbContent[4] = "2.openGaussScore" tbContent[5] = i.OpenEulerScore tbContent[6] = "已分析" - tbContent[7] = "openGaussVector" + tbContent[7] = "2.openGaussVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } else { if i.OpenEulerScore == 0.0 { - msg = fmt.Sprintf("openGaussScore没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.openGaussScore=> 没有填写或正确填写(0-10)") ok = false return } tbContent[3] = "已分析" - tbContent[4] = "openGaussScore" + tbContent[4] = "2.openGaussScore" tbContent[5] = i.OpenEulerScore if i.OpenEulerVector == "" || len(i.OpenEulerVector) < 1 { - msg = fmt.Sprintf("openGaussVector没有正确填写") + msg = fmt.Sprintf("2.openGaussVector=> 没有正确填写") ok = false return } tbContent[6] = "已分析" - tbContent[7] = "openGaussVector" + tbContent[7] = "2.openGaussVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } tbStr = fmt.Sprintf(tb, tbContent...) @@ -2243,12 +2249,108 @@ func checkSporeIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string ok = true tbContent := make([]interface{}, 12) if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { - msg = fmt.Sprintf("影响性分析说明 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + return + } + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + return + } + if versionfFlag { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } else { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.MindSporeScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.MindSporeVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + msg = fmt.Sprintf("2.MindSporeScore=> 没有填写或正确填写(0-10)") + ok = false + return + } + tbContent[3] = "已分析" + tbContent[4] = "2.MindSporeScore" + tbContent[5] = i.OpenEulerScore + if i.OpenEulerVector == "" || len(i.OpenEulerVector) < 1 { + msg = fmt.Sprintf("2.MindSporeVector=> 没有正确填写") + ok = false + return + } + tbContent[6] = "已分析" + tbContent[7] = "2.MindSporeVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func checkLooKengIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") ok = false return } tbContent[0] = "已分析" - tbContent[1] = "影响性分析说明" + tbContent[1] = "1.影响性分析说明" tbContent[2] = util.TrimStringNR(i.CveAnalysis) affectedVersionFlag := 1 if i.AffectedVersion != "" { @@ -2276,47 +2378,47 @@ func checkSporeIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string } } if !versionfFlag { - msg = fmt.Sprintf("受影响版本排查(受影响/不受影响): 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) ok = false return } if versionfFlag { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = util.TrimStringNR(i.AffectedVersion) } else { tbContent[9] = "待分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = util.TrimStringNR(i.AffectedVersion) } } else { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = "" } if affectedVersionFlag == 1 { tbContent[3] = "已分析" - tbContent[4] = "MindSporeScore" + tbContent[4] = "2.openLooKengScore" tbContent[5] = i.OpenEulerScore tbContent[6] = "已分析" - tbContent[7] = "MindSporeVector" + tbContent[7] = "2.openLooKengVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } else { if i.OpenEulerScore == 0.0 { - msg = fmt.Sprintf("MindSporeScore没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.openLooKengScore=> 没有填写或正确填写(0-10)") ok = false return } tbContent[3] = "已分析" - tbContent[4] = "MindSporeScore" + tbContent[4] = "2.openLooKengScore" tbContent[5] = i.OpenEulerScore if i.OpenEulerVector == "" || len(i.OpenEulerVector) < 1 { - msg = fmt.Sprintf("MindSporeVector没有正确填写") + msg = fmt.Sprintf("2.openLooKengVector=> 没有正确填写") ok = false return } tbContent[6] = "已分析" - tbContent[7] = "MindSporeVector" + tbContent[7] = "2.openLooKengVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } tbStr = fmt.Sprintf(tb, tbContent...) @@ -2339,15 +2441,15 @@ func checkIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr strin } ok = true tbContent := make([]interface{}, 15) - if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 2 { tbContent[0] = "待分析" - tbContent[1] = "影响性分析说明" - tbContent[2] = fmt.Sprintf("影响性分析说明 没有填写或按正确格式填写") - msg = fmt.Sprintf("影响性分析说明 没有填写或按正确格式填写") + tbContent[1] = "1.影响性分析说明" + tbContent[2] = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") ok = false } else { tbContent[0] = "已分析" - tbContent[1] = "影响性分析说明" + tbContent[1] = "1.影响性分析说明" tbContent[2] = util.TrimStringNR(i.CveAnalysis) } affectedVersionFlag := 1 @@ -2378,48 +2480,48 @@ func checkIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr strin } if !versionfFlag { tbContent[9] = "待分析" - tbContent[10] = "受影响版本排查" - tbContent[11] = fmt.Sprintf("受影响版本排查(受影响/不受影响): 没有分析或未按正确格式填写:%v", i.AffectedVersion) - msg = fmt.Sprintf("受影响版本排查(受影响/不受影响): 没有分析或未按正确格式填写:%v", i.AffectedVersion) + tbContent[10] = "3.受影响版本排查" + tbContent[11] = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) ok = false } else { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = util.TrimStringNR(i.AffectedVersion) } } else { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = "" } if affectedVersionFlag == 1 { tbContent[3] = "已分析" - tbContent[4] = "openEulerScore" + tbContent[4] = "2.openEulerScore" tbContent[5] = i.OpenEulerScore tbContent[6] = "已分析" - tbContent[7] = "openEulerVector" + tbContent[7] = "2.openEulerVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } else { if i.OpenEulerScore == 0.0 { tbContent[3] = "待分析" - tbContent[4] = "openEulerScore" - tbContent[5] = fmt.Sprintf("openEulerScore没有填写或正确填写(0-10)") - msg = fmt.Sprintf("openEulerScore没有填写或正确填写(0-10)") + tbContent[4] = "2.openEulerScore" + tbContent[5] = fmt.Sprintf("2.openEulerScore=> 没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.openEulerScore=> 没有填写或正确填写(0-10)") ok = false } else { tbContent[3] = "已分析" - tbContent[4] = "openEulerScore" + tbContent[4] = "2.openEulerScore" tbContent[5] = i.OpenEulerScore } if util.TrimString(i.OpenEulerVector) == "" || len(util.TrimString(i.OpenEulerVector)) < 1 { tbContent[6] = "待分析" - tbContent[7] = "openEulerVector" - tbContent[8] = fmt.Sprintf("openEulerVector没有正确填写") - msg = fmt.Sprintf("openEulerVector没有正确填写") + tbContent[7] = "2.openEulerVector" + tbContent[8] = fmt.Sprintf("2.openEulerVector=> 没有正确填写") + msg = fmt.Sprintf("2.openEulerVector=> 没有正确填写") ok = false } else { tbContent[6] = "已分析" - tbContent[7] = "openEulerVector" + tbContent[7] = "2.openEulerVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } } @@ -2445,18 +2547,18 @@ func checkIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr strin } if !versionAbiFlag { tbContent[12] = "待分析" - tbContent[13] = "修复是否涉及abi变化" - tbContent[14] = fmt.Sprintf("修复是否涉及abi变化(是/否): 没有分析或未按正确格式填写:%v", i.AbiVersion) - msg = fmt.Sprintf("修复是否涉及abi变化(是/否): 没有分析或未按正确格式填写:%v", i.AbiVersion) + tbContent[13] = "4.修复是否涉及abi变化" + tbContent[14] = fmt.Sprintf("4.修复是否涉及abi变化(是/否)=> 没有分析或未按正确格式填写:%v", i.AbiVersion) + msg = fmt.Sprintf("4.修复是否涉及abi变化(是/否)=> 没有分析或未按正确格式填写:%v", i.AbiVersion) ok = false } else { tbContent[12] = "已分析" - tbContent[13] = "修复是否涉及abi变化" + tbContent[13] = "4.修复是否涉及abi变化" tbContent[14] = util.TrimStringNR(i.AbiVersion) } } else { tbContent[12] = "已分析" - tbContent[13] = "修复是否涉及abi变化" + tbContent[13] = "4.修复是否涉及abi变化" tbContent[14] = "" } tbStr = fmt.Sprintf(tb, tbContent...) @@ -2478,15 +2580,15 @@ func checkGaussIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr } ok = true tbContent := make([]interface{}, 12) - if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 2 { tbContent[0] = "待分析" - tbContent[1] = "影响性分析说明" - tbContent[2] = fmt.Sprintf("影响性分析说明 没有填写或按正确格式填写") - msg = fmt.Sprintf("影响性分析说明 没有填写或按正确格式填写") + tbContent[1] = "1.影响性分析说明" + tbContent[2] = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") ok = false } else { tbContent[0] = "已分析" - tbContent[1] = "影响性分析说明" + tbContent[1] = "1.影响性分析说明" tbContent[2] = util.TrimStringNR(i.CveAnalysis) } affectedVersionFlag := 1 @@ -2516,48 +2618,48 @@ func checkGaussIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr } if !versionfFlag { tbContent[9] = "待分析" - tbContent[10] = "受影响版本排查" - tbContent[11] = fmt.Sprintf("受影响版本排查(受影响/不受影响): 没有分析或未按正确格式填写:%v", i.AffectedVersion) - msg = fmt.Sprintf("受影响版本排查(受影响/不受影响): 没有分析或未按正确格式填写:%v", i.AffectedVersion) + tbContent[10] = "3.受影响版本排查" + tbContent[11] = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) ok = false } else { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = util.TrimStringNR(i.AffectedVersion) } } else { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = "" } if affectedVersionFlag == 1 { tbContent[3] = "已分析" - tbContent[4] = "openGaussScore" + tbContent[4] = "2.openGaussScore" tbContent[5] = i.OpenEulerScore tbContent[6] = "已分析" - tbContent[7] = "openGaussVector" + tbContent[7] = "2.openGaussVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } else { if i.OpenEulerScore == 0.0 { tbContent[3] = "待分析" - tbContent[4] = "openGaussScore" - tbContent[5] = fmt.Sprintf("openGaussScore没有填写或正确填写(0-10)") - msg = fmt.Sprintf("openGaussScore没有填写或正确填写(0-10)") + tbContent[4] = "2.openGaussScore" + tbContent[5] = fmt.Sprintf("2.openGaussScore=> 没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.openGaussScore=> 没有填写或正确填写(0-10)") ok = false } else { tbContent[3] = "已分析" - tbContent[4] = "openGaussScore" + tbContent[4] = "2.openGaussScore" tbContent[5] = i.OpenEulerScore } if util.TrimString(i.OpenEulerVector) == "" || len(util.TrimString(i.OpenEulerVector)) < 1 { tbContent[6] = "待分析" - tbContent[7] = "openGaussVector" - tbContent[8] = fmt.Sprintf("openGaussVector没有正确填写") - msg = fmt.Sprintf("openGaussVector没有正确填写") + tbContent[7] = "2.openGaussVector" + tbContent[8] = fmt.Sprintf("2.openGaussVector=> 没有正确填写") + msg = fmt.Sprintf("2.openGaussVector=> 没有正确填写") ok = false } else { tbContent[6] = "已分析" - tbContent[7] = "openGaussVector" + tbContent[7] = "2.openGaussVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } } @@ -2580,15 +2682,15 @@ func checkSporeIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr } ok = true tbContent := make([]interface{}, 12) - if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 2 { tbContent[0] = "待分析" - tbContent[1] = "影响性分析说明" - tbContent[2] = fmt.Sprintf("影响性分析说明 没有填写或按正确格式填写") - msg = fmt.Sprintf("影响性分析说明 没有填写或按正确格式填写") + tbContent[1] = "1.影响性分析说明" + tbContent[2] = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") ok = false } else { tbContent[0] = "已分析" - tbContent[1] = "影响性分析说明" + tbContent[1] = "1.影响性分析说明" tbContent[2] = util.TrimStringNR(i.CveAnalysis) } affectedVersionFlag := 1 @@ -2618,48 +2720,150 @@ func checkSporeIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr } if !versionfFlag { tbContent[9] = "待分析" - tbContent[10] = "受影响版本排查" - tbContent[11] = fmt.Sprintf("受影响版本排查(受影响/不受影响): 没有分析或未按正确格式填写:%v", i.AffectedVersion) - msg = fmt.Sprintf("受影响版本排查(受影响/不受影响): 没有分析或未按正确格式填写:%v", i.AffectedVersion) + tbContent[10] = "3.受影响版本排查" + tbContent[11] = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) ok = false } else { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = util.TrimStringNR(i.AffectedVersion) } } else { tbContent[9] = "已分析" - tbContent[10] = "受影响版本排查" + tbContent[10] = "3.受影响版本排查" tbContent[11] = "" } if affectedVersionFlag == 1 { tbContent[3] = "已分析" - tbContent[4] = "MindSporeScore" + tbContent[4] = "2.MindSporeScore" tbContent[5] = i.OpenEulerScore tbContent[6] = "已分析" - tbContent[7] = "MindSporeVector" + tbContent[7] = "2.MindSporeVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } else { if i.OpenEulerScore == 0.0 { tbContent[3] = "待分析" - tbContent[4] = "MindSporeScore" - tbContent[5] = fmt.Sprintf("MindSporeScore没有填写或正确填写(0-10)") - msg = fmt.Sprintf("MindSporeScore没有填写或正确填写(0-10)") + tbContent[4] = "2.MindSporeScore" + tbContent[5] = fmt.Sprintf("2.MindSporeScore=> 没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.MindSporeScore=> 没有填写或正确填写(0-10)") ok = false } else { tbContent[3] = "已分析" - tbContent[4] = "MindSporeScore" + tbContent[4] = "2.MindSporeScore" tbContent[5] = i.OpenEulerScore } if util.TrimString(i.OpenEulerVector) == "" || len(util.TrimString(i.OpenEulerVector)) < 1 { tbContent[6] = "待分析" - tbContent[7] = "MindSporeVector" - tbContent[8] = fmt.Sprintf("MindSporeVector没有正确填写") - msg = fmt.Sprintf("MindSporeVector没有正确填写") + tbContent[7] = "2.MindSporeVector" + tbContent[8] = fmt.Sprintf("2.MindSporeVector=> 没有正确填写") + msg = fmt.Sprintf("2.MindSporeVector=> 没有正确填写") ok = false } else { tbContent[6] = "已分析" - tbContent[7] = "MindSporeVector" + tbContent[7] = "2.MindSporeVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func checkLooKengIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 2 { + tbContent[0] = "待分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + msg = fmt.Sprintf("1.影响性分析说明=> 没有填写或按正确格式填写") + ok = false + } else { + tbContent[0] = "已分析" + tbContent[1] = "1.影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + } + affectedVersionFlag := 1 + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + if versionArry[1] == "受影响" { + affectedVersionFlag = 2 + } + continue + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } else { + affectedVersionFlag = 3 + versionfFlag = false + break + } + } + } + if !versionfFlag { + tbContent[9] = "待分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("3.受影响版本排查(受影响/不受影响)=> 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "3.受影响版本排查" + tbContent[11] = "" + } + if affectedVersionFlag == 1 { + tbContent[3] = "已分析" + tbContent[4] = "2.openLooKengScore" + tbContent[5] = i.OpenEulerScore + tbContent[6] = "已分析" + tbContent[7] = "2.openLooKengVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } else { + if i.OpenEulerScore == 0.0 { + tbContent[3] = "待分析" + tbContent[4] = "2.openLooKengScore" + tbContent[5] = fmt.Sprintf("2.openLooKengScore=> 没有填写或正确填写(0-10)") + msg = fmt.Sprintf("2.openLooKengScore=> 没有填写或正确填写(0-10)") + ok = false + } else { + tbContent[3] = "已分析" + tbContent[4] = "2.openLooKengScore" + tbContent[5] = i.OpenEulerScore + } + if util.TrimString(i.OpenEulerVector) == "" || len(util.TrimString(i.OpenEulerVector)) < 1 { + tbContent[6] = "待分析" + tbContent[7] = "2.openLooKengVector" + tbContent[8] = fmt.Sprintf("2.openLooKengVector=> 没有正确填写") + msg = fmt.Sprintf("2.openLooKengVector=> 没有正确填写") + ok = false + } else { + tbContent[6] = "已分析" + tbContent[7] = "2.openLooKengVector" tbContent[8] = util.TrimStringNR(i.OpenEulerVector) } } @@ -2855,7 +3059,7 @@ func AddGitIssue(issueHook *models.IssuePayload, desc, product string) error { if strings.HasPrefix(issueTitle, "CVE") { cveStr = issueTitle } else if issueHook.Issue.Body != "" { - //通过正则表达式去body中截取 稍后完善 + // Use regular expressions to intercept the body, which will be improved later sm := util.RegexpCveNumber.FindAllStringSubmatch(issueHook.Issue.Body, -1) if len(sm) > 0 && len(sm[0]) > 0 { cveStr = sm[0][1] @@ -2875,15 +3079,7 @@ func AddGitIssue(issueHook *models.IssuePayload, desc, product string) error { } } } - owner := beego.AppConfig.String("gitee::owner") - accessToken := beego.AppConfig.String("gitee::git_token") - if organizationID == 3 { - owner = beego.AppConfig.String("mindspore::mindspore_owner") - accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") - } else if organizationID == 2 { - owner = beego.AppConfig.String("opengauss::gauss_owner") - accessToken = beego.AppConfig.String("opengauss::git_gauss_token") - } + owner, accessToken := common.GetOwnerAndToken(organizationID) for _, cve := range cveList { item.CveNumber = cve // Check whether the current cve has created an issue @@ -2910,7 +3106,7 @@ func AddGitIssue(issueHook *models.IssuePayload, desc, product string) error { } } -func DelOrgIssue(issueHook *models.IssuePayload) { +func DelOrgIssue(issueHook *models.IssuePayload, organizationID int8) { issueTitle := util.TrimString(issueHook.Title) issueType := util.TrimString(issueHook.Issue.TypeName) issueNumber := util.TrimString(issueHook.Issue.Number) @@ -2918,7 +3114,6 @@ func DelOrgIssue(issueHook *models.IssuePayload) { issueZhState := util.TrimString(issueHook.Issue.StateName) repoPath := util.TrimString(issueHook.Repository.Path) nameSpace := util.TrimString(issueHook.Repository.NameSpace) - organizationID := int8(1) organizationID = taskhandler.GetOrganizationId(nameSpace) if issueType == CIssueType || strings.HasPrefix(issueTitle, "CVE") { // Data deletion record @@ -2976,9 +3171,8 @@ func DelOrgIssue(issueHook *models.IssuePayload) { } // Entry function for handling issue status -func gitAddIssueProc(issueHook *models.IssuePayload) error { - token := os.Getenv("GITEE_TOKEN") - owner := beego.AppConfig.String("gitee::owner") +func gitAddIssueProc(issueHook *models.IssuePayload, organizationID int8) error { + owner, token := common.GetOwnerAndToken(organizationID) path := issueHook.Repository.Path // The amount of data processed at a time prcNum, err := beego.AppConfig.Int("crontab::prcnum") @@ -3042,14 +3236,31 @@ func AddIssueComment(token, owner, path, issueNum, assignee string, issueId int6 if cveErr != nil { return cveErr } - if cveCenter.OrganizationID == 3 { - owner = beego.AppConfig.String("mindspore::mindspore_owner") - token = beego.AppConfig.String("mindspore::git_mindspore_token") + owner, token = common.GetOwnerAndToken(cveCenter.OrganizationID) + if cveCenter.OrganizationID == 4 { + cveList := strings.Split(cveCenter.CveVersion, ",") + if len(cveList) > 0 { + for _, vl := range cveList { + olky := models.OpenLookengYaml{PackageName: cveCenter.PackName, Version: vl} + looKengErr := models.GetOpenLookengYaml(&olky, "PackageName", "Version") + if olky.Id > 0 { + path = olky.Repo + break + } + logs.Info("GetOpenLookengYaml, looKengErr: ", looKengErr) + } + } + branchList = taskhandler.CreateBrandAndTags(token, owner, path, cveCenter.OrganizationID) + if branchList == nil || len(branchList) == 0 { + logs.Error("OpenLookeng GetBranchesInfo, Failed to obtain the branch information of the repo, ", path, ", err: ", errBrands) + return errors.New("Failed to obtain branch information") + } + } else if cveCenter.OrganizationID == 3 { // Query the repo that needs to submit an issue cveList := strings.Split(cveCenter.CveVersion, ",") if len(cveList) > 0 { - for _, cl := range cveList { - ms := models.MindSporeYaml{PackageName: cveCenter.PackName, Version: cl} + for _, vl := range cveList { + ms := models.MindSporeYaml{PackageName: cveCenter.PackName, Version: vl} sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") if ms.Id > 0 { path = ms.Repo @@ -3065,8 +3276,6 @@ func AddIssueComment(token, owner, path, issueNum, assignee string, issueId int6 return errors.New("Failed to obtain branch information") } } else if cveCenter.OrganizationID == 2 { - owner = beego.AppConfig.String("opengauss::gauss_owner") - token = beego.AppConfig.String("opengauss::git_gauss_token") path = beego.AppConfig.String("opengauss::openGauss-server") // Get branch information branchList, errBrands = taskhandler.GetBranchesInfo(token, owner, path, cveCenter.OrganizationID) @@ -3107,7 +3316,7 @@ func AddIssueComment(token, owner, path, issueNum, assignee string, issueId int6 return errx } -func gitDelIssueProc(issueHook *models.IssuePayload) error { - DelOrgIssue(issueHook) +func gitDelIssueProc(issueHook *models.IssuePayload, organizationID int8) error { + DelOrgIssue(issueHook, organizationID) return nil } diff --git a/cve-vulner-manager/cve-py/config_yaml/deployment.yaml b/cve-vulner-manager/cve-py/config_yaml/deployment.yaml new file mode 100644 index 0000000..646ce9c --- /dev/null +++ b/cve-vulner-manager/cve-py/config_yaml/deployment.yaml @@ -0,0 +1,44 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + labels: + app: cve-manager-analysis + name: cve-manager-analysis +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: cve-manager-analysis + template: + metadata: + labels: + app: cve-manager-analysis + spec: + containers: + - env: + - name: DB_PWDPY + valueFrom: + secretKeyRef: + name: cve-secrets-py + key: db-pwdpy + - name: DB_URI + valueFrom: + secretKeyRef: + name: cve-secrets-py + key: db-uri + - name: CVE_EMAIL_SENDADDR + valueFrom: + secretKeyRef: + name: cve-secrets-py + key: cve-email-sendaddr + - name: CVE_EMAIL_PASSWORD + valueFrom: + secretKeyRef: + name: cve-secrets-py + key: cve-email-password + - name: TZ + value: Asia/Shanghai + image: swr.cn-north-4.myhuaweicloud.com/opensourceway/openeuler/cve-manager-analysis:1cb6af4c1d428074cb8e54db23adf1efaab75639 + imagePullPolicy: IfNotPresent + name: cve-manager-analysis diff --git a/cve-vulner-manager/cve-py/config_yaml/kustomization.yaml b/cve-vulner-manager/cve-py/config_yaml/kustomization.yaml new file mode 100644 index 0000000..9684b1e --- /dev/null +++ b/cve-vulner-manager/cve-py/config_yaml/kustomization.yaml @@ -0,0 +1,17 @@ +resources: +- namespace.yaml +- deployment.yaml +- service.yaml +- secrets.yaml +commonLabels: + app: cve-manager-analysis + owner: zhangjianjun +commonAnnotations: + email: 841670711@qq.com + owner: zhangjianjun +namespace: cve-manager +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: swr.cn-north-4.myhuaweicloud.com/opensourceway/openeuler/cve-manager-analysis + newTag: 89281957eaa42c6a7c8cd048b1374dccdf68fd96 diff --git a/cve-vulner-manager/cve-py/config_yaml/namespace.yaml b/cve-vulner-manager/cve-py/config_yaml/namespace.yaml new file mode 100644 index 0000000..09d5c2b --- /dev/null +++ b/cve-vulner-manager/cve-py/config_yaml/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + name: cve-manager-analysis + name: cve-manager-analysis diff --git a/cve-vulner-manager/cve-py/config_yaml/secrets.yaml b/cve-vulner-manager/cve-py/config_yaml/secrets.yaml new file mode 100644 index 0000000..eb2c021 --- /dev/null +++ b/cve-vulner-manager/cve-py/config_yaml/secrets.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: secrets-manager.tuenti.io/v1alpha1 +kind: SecretDefinition +metadata: + name: cve-secrets-py +spec: + name: cve-secrets-py + keysMap: + db-pwdpy: + path: secrets/data/openeuler/cve-manager + key: db-pwdpy + db-uri: + path: secrets/data/openeuler/cve-manager + key: db-uri + cve-email-sendaddr: + path: secrets/data/openeuler/cve-manager + key: cve-email-sendaddr + cve-email-password: + path: secrets/data/openeuler/cve-manager + key: cve-email-password diff --git a/cve-vulner-manager/cve-py/config_yaml/service.yaml b/cve-vulner-manager/cve-py/config_yaml/service.yaml new file mode 100644 index 0000000..5d38198 --- /dev/null +++ b/cve-vulner-manager/cve-py/config_yaml/service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: cve-manager-analysis-service + namespace: cve-manager-analysis +spec: + ports: + - name: cve-manager-analysis + port: 8080 + protocol: TCP + targetPort: 8080 + selector: + app: cve-manager-analysis + type: ClusterIP diff --git a/cve-vulner-manager/cve-py/controller/timertaskcontroller.py b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py index 6404323..8f53059 100644 --- a/cve-vulner-manager/cve-py/controller/timertaskcontroller.py +++ b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py @@ -41,13 +41,13 @@ def timertask(): scheduler.add_job(taskcontroller.run_whiltlist_task, 'interval', hours=2) # parsing the table package_committer_excels scheduler.add_job(taskcontroller.run_package_committer_task, 'interval', hours=1) - # Parse opengauss yaml file - scheduler.add_job(taskcontroller.parse_opengauss_yaml_task, 'cron', day_of_week='0-6', hour=2, minute=30) # Parse the issue statistics recipient list # scheduler.add_job(taskcontroller.issue_statistics_email_task, 'cron', day_of_week='0-6', hour=5, minute=30) # Complete the template information of the issue with the data on the CVE official website scheduler.add_job(taskcontroller.supplement_cve_task, 'interval', minutes=20) scheduler.add_job(taskcontroller.long_supplement_cve_task, 'cron', day_of_week='0-6', hour=1, minute=30) + # Parse opengauss yaml file + scheduler.add_job(taskcontroller.parse_opengauss_yaml_task, 'cron', day_of_week='0-6', hour=2, minute=30) # Parse the yaml file of mindspore scheduler.add_job(taskcontroller.parse_mindspore_yaml_task, 'cron', day_of_week='0-6', hour=3, minute=30) # Parse the yaml file of openLooKeng diff --git a/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py b/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py index 1bc8f7c..c906254 100644 --- a/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py +++ b/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py @@ -58,16 +58,16 @@ def store_yaml_data(yaml_data): mysql = Mysql() if yaml_data is not None and len(yaml_data) > 0: # Delete all data before updating - delete_yaml_origin_data(mysql) - origin_id = 1 + update_yaml_origin_mark(mysql) for yaml_key, yaml_value in yaml_data.items(): try: + if "version" not in yaml_value or not yaml_value["version"]: + yaml_value["version"] = "" origin_data = select_yaml_origin_data(yaml_key, yaml_value["version"], mysql) if origin_data: update_yaml_origin_data(origin_data["id"], yaml_value, mysql) else: - insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id) - origin_id += 1 + insert_yaml_origin_data(yaml_key, yaml_value, mysql) packages_data = select_yaml_data(yaml_key, yaml_value, mysql) if packages_data is not None: if len(packages_data) > 1: @@ -91,6 +91,7 @@ def store_yaml_data(yaml_data): # except Exception as e: # print(e) mysql.dispose(2) + delete_yaml_origin_mark(mysql) mysql.close() @@ -184,23 +185,23 @@ def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): Update origin data """ update_sql = "update cve_open_guss_yaml set origin_url = %s,status = %s, " \ - "cpe_name = %s, update_time = %s,repo_name=%s where id = %s" + "cpe_name = %s, update_time = %s,repo_name=%s, mark_bit = %s where id = %s" val = (yaml_value["url"], 1, yaml_value["cpeName"], - times.get_current_time(), "security", pg_origin_id) + times.get_current_time(), "security", 1, pg_origin_id) mysql.update(update_sql, val) mysql.dispose() -def insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id): +def insert_yaml_origin_data(yaml_key, yaml_value, mysql): """ insert origin data """ insert_sql = "insert into cve_open_guss_yaml(package_name," \ - "version,origin_url, status, cpe_name,create_time,update_time, id, repo_name) " \ + "version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) " \ "values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" val = (yaml_key, yaml_value["version"], yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), - times.get_current_time(), origin_id, "security") + times.get_current_time(), 1, "security") last_id = mysql.insertOne(insert_sql, val) mysql.dispose() return last_id @@ -221,11 +222,20 @@ def select_yaml_data(yaml_key, yaml_value, mysql): return None -def delete_yaml_origin_data(mysql): +def update_yaml_origin_mark(mysql): """ Update origin data """ - delete_sql = "truncate table cve_open_guss_yaml" + update_sql = "update cve_open_guss_yaml set mark_bit = 2 where mark_bit = 1" + mysql.update(update_sql) + mysql.dispose() + + +def delete_yaml_origin_mark(mysql): + """ + delete origin data + """ + delete_sql = "delete from cve_open_guss_yaml where mark_bit = 2" mysql.delete(delete_sql) mysql.dispose() diff --git a/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py b/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py index 4545094..0118ebd 100644 --- a/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py +++ b/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py @@ -138,23 +138,23 @@ def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): Update origin data """ update_sql = "update cve_mind_spore_yaml set origin_url = %s,status = %s, " \ - "cpe_name = %s, update_time = %s where id = %s" + "cpe_name = %s, update_time = %s, mark_bit = %s where id = %s" val = (yaml_value["url"], 1, yaml_value["cpeName"], - times.get_current_time(), pg_origin_id) + times.get_current_time(), 1, pg_origin_id) mysql.update(update_sql, val) mysql.dispose() -def insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id, repo_key): +def insert_yaml_origin_data(yaml_key, yaml_value, mysql, repo_key): """ insert origin data """ insert_sql = "insert into cve_mind_spore_yaml(package_name," \ - "version,origin_url, status, cpe_name,create_time,update_time, id, repo_name) " \ + "version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) " \ "values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" val = (yaml_key, yaml_value["version"], yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), - times.get_current_time(), origin_id, repo_key) + times.get_current_time(), 1, repo_key) last_id = mysql.insertOne(insert_sql, val) mysql.dispose() return last_id @@ -175,11 +175,20 @@ def select_yaml_data(yaml_key, yaml_value, mysql): return None -def delete_yaml_origin_data(mysql): +def update_yaml_origin_mark(mysql): """ Update origin data """ - delete_sql = "truncate table cve_mind_spore_yaml" + update_sql = "update cve_mind_spore_yaml set mark_bit = 2 where mark_bit = 1" + mysql.update(update_sql) + mysql.dispose() + + +def delete_yaml_origin_mark(mysql): + """ + delete origin data + """ + delete_sql = "delete from cve_mind_spore_yaml where mark_bit = 2" mysql.delete(delete_sql) mysql.dispose() @@ -220,17 +229,17 @@ def store_yaml_data(yaml_data): mysql = Mysql() if yaml_data is not None and len(yaml_data) > 0: # Delete all data before updating - delete_yaml_origin_data(mysql) - origin_id = 1 + update_yaml_origin_mark(mysql) for repo_key, repo_value in yaml_data.items(): try: for yaml_key, yaml_value in repo_value.items(): + if "version" not in yaml_value or not yaml_value["version"]: + yaml_value["version"] = "" origin_data = select_yaml_origin_data(yaml_key, yaml_value["version"], mysql, repo_key) if origin_data: update_yaml_origin_data(origin_data["id"], yaml_value, mysql) else: - insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id, repo_key) - origin_id += 1 + insert_yaml_origin_data(yaml_key, yaml_value, mysql, repo_key) packages_data = select_yaml_data(yaml_key, yaml_value, mysql) if packages_data is not None: if len(packages_data) > 1: @@ -254,6 +263,7 @@ def store_yaml_data(yaml_data): # except Exception as e: # print(e) mysql.dispose(2) + delete_yaml_origin_mark(mysql) mysql.close() diff --git a/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py b/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py index bd04bf1..a4f5c95 100644 --- a/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py +++ b/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py @@ -31,8 +31,8 @@ def download_openlookeng_yaml(): file_name = file_path + 'openlookeng_yaml.yaml' if mk_ok: # download the yaml file - file_url = 'https://gitee.com/mindspore/community/raw/' \ - 'master/security/config/Third_Party_Open_Source_Software_List.yaml' + file_url = 'https://gitee.com/openlookeng/community/raw/master/security' \ + '/config/Third_Party_Open_Source_Software_List.yaml' downloadfiletask.download_yaml(file_name, file_url) # pass return file_name @@ -138,23 +138,23 @@ def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): Update origin data """ update_sql = "update cve_open_lookeng_yaml set origin_url = %s,status = %s, " \ - "cpe_name = %s, update_time = %s where id = %s" + "cpe_name = %s, update_time = %s,mark_bit = %s where id = %s" val = (yaml_value["url"], 1, yaml_value["cpeName"], - times.get_current_time(), pg_origin_id) + times.get_current_time(), 1, pg_origin_id) mysql.update(update_sql, val) mysql.dispose() -def insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id, repo_key): +def insert_yaml_origin_data(yaml_key, yaml_value, mysql, repo_key): """ insert origin data """ insert_sql = "insert into cve_open_lookeng_yaml(package_name," \ - "version,origin_url, status, cpe_name,create_time,update_time, id, repo_name) " \ + "version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) " \ "values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" val = (yaml_key, yaml_value["version"], yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), - times.get_current_time(), origin_id, repo_key) + times.get_current_time(), 1, repo_key) last_id = mysql.insertOne(insert_sql, val) mysql.dispose() return last_id @@ -175,12 +175,21 @@ def select_yaml_data(yaml_key, yaml_value, mysql): return None -def delete_yaml_origin_data(mysql): +def update_yaml_origin_mark(mysql): """ Update origin data """ - delete_sql = "delete from cve_open_lookeng_yaml" - mysql.delete(delete_sql) + update_sql = "update cve_open_lookeng_yaml set mark_bit = 2 where mark_bit = 1" + mysql.update(update_sql) + mysql.dispose() + + +def delete_yaml_origin_mark(mysql): + """ + delete origin data + """ + delete_sql = "delete from cve_open_lookeng_yaml where mark_bit = 2" + mysql.update(delete_sql) mysql.dispose() @@ -220,17 +229,17 @@ def store_yaml_data(yaml_data): mysql = Mysql() if yaml_data is not None and len(yaml_data) > 0: # Delete all data before updating - delete_yaml_origin_data(mysql) - origin_id = 1 + update_yaml_origin_mark(mysql) for repo_key, repo_value in yaml_data.items(): try: for yaml_key, yaml_value in repo_value.items(): + if "version" not in yaml_value or not yaml_value["version"]: + yaml_value["version"] = "" origin_data = select_yaml_origin_data(yaml_key, yaml_value["version"], mysql, repo_key) if origin_data: update_yaml_origin_data(origin_data["id"], yaml_value, mysql) else: - insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id, repo_key) - origin_id += 1 + insert_yaml_origin_data(yaml_key, yaml_value, mysql, repo_key) packages_data = select_yaml_data(yaml_key, yaml_value, mysql) if packages_data is not None: if len(packages_data) > 1: @@ -254,6 +263,7 @@ def store_yaml_data(yaml_data): # except Exception as e: # print(e) mysql.dispose(2) + delete_yaml_origin_mark(mysql) mysql.close() diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go index 67d9f17..333a267 100644 --- a/cve-vulner-manager/models/issue.go +++ b/cve-vulner-manager/models/issue.go @@ -666,6 +666,12 @@ func GetCommunityYamlConfigAll() (cyc []CommunityYamlConfig) { return } +func InsertCommunityYamlConfig(cyc *CommunityYamlConfig) (int64, error) { + o := orm.NewOrm() + id, err := o.Insert(cyc) + return id, err +} + type CommunityPackage struct { Id int64 `orm:"column(id)"` PackageName string `orm:"column(package_name)" description:"组件名称"` diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index cbd20bb..c0e51c2 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -50,7 +50,7 @@ type VulnCenter struct { IsExport int8 `orm:"default(0);column(is_export)" description:"0:数据初始化;1:数据已同步到官网；2：数据不需要同步到官网；3：issue已关闭需要同步到官网"` DataSource int8 `orm:"default(1);column(data_source)" description:"1:软件所提供;2:网络爬取；3:人工处理; 4:兼容之前的数据"` CveDetailUrl string `orm:"type(text);column(cve_detail_url)" description:"cve详情链接"` - OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore"` + OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore;4:来源openLooKeng"` CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` @@ -510,7 +510,7 @@ type GiteOriginIssue struct { RepoDesc string `json:"repo_desc" orm:"null;size(2048)" description:"cve 对组件的描述"` IssueState string `orm:"size(50);column(issue_state)" description:"issue 中文状态"` Owner string `orm:"size(256);column(owner);default(src-openeuler)" description:"issue所属社区"` - OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源src-openEuler;2:来源opengauss;3:来源mindspore"` + OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源src-openEuler;2:来源opengauss;3:来源mindspore;4:来源openLooKeng"` } type OriginExcel struct { @@ -642,6 +642,7 @@ type OpenGussYaml struct { OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` CpeName string `orm:"type(text);column(cpe_name)" description:"cpe的名称"` + MarkBit int8 `orm:"default(1);column(mark_bit)" description:"1:正常;2:已删除"` CreateTime string `orm:"size(32);column(create_time)"` UpdateTime string `orm:"size(32);column(update_time);null"` DeleteTime string `orm:"size(32);column(delete_time);null"` @@ -892,6 +893,7 @@ type MindSporeYaml struct { OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` CpeName string `orm:"type(text);column(cpe_name)" description:"cpe的名称"` + MarkBit int8 `orm:"default(1);column(mark_bit)" description:"1:正常;2:已删除"` CreateTime string `orm:"size(32);column(create_time)"` UpdateTime string `orm:"size(32);column(update_time);null"` DeleteTime string `orm:"size(32);column(delete_time);null"` @@ -956,6 +958,7 @@ type OpenLookengYaml struct { OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` CpeName string `orm:"type(text);column(cpe_name)" description:"cpe的名称"` + MarkBit int8 `orm:"default(1);column(mark_bit)" description:"1:正常;2:已删除"` CreateTime string `orm:"size(32);column(create_time)"` UpdateTime string `orm:"size(32);column(update_time);null"` DeleteTime string `orm:"size(32);column(delete_time);null"` @@ -977,7 +980,7 @@ type IssueCommunityStatistics struct { type CommunityYamlConfig struct { Id int64 `orm:"pk;auto"` TableName string `orm:"column(table_name);size(512)" description:"表名称"` - OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore"` + OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore;4:来源openLooKeng"` } type IssueDeleteRecord struct { diff --git a/cve-vulner-manager/models/reviewer.go b/cve-vulner-manager/models/reviewer.go index 437e29c..1739545 100644 --- a/cve-vulner-manager/models/reviewer.go +++ b/cve-vulner-manager/models/reviewer.go @@ -43,6 +43,12 @@ func GetSporeSecurityReviewerList() (list []MindSporeSecurityReviewer, err error return list, err } +func GetOpenLookengSecurityReviewerList() (list []OpenLookengSecurityReviewer, err error) { + o := orm.NewOrm() + _, err = o.QueryTable("cve_open_lookeng_security_reviewer").All(&list) + return list, err +} + func (s *MindSporeSecurityReviewer) Read(filed ...string) (ok bool) { o := orm.NewOrm() err := o.Read(s, filed...) @@ -51,4 +57,14 @@ func (s *MindSporeSecurityReviewer) Read(filed ...string) (ok bool) { return false } return true +} + +func (s *OpenLookengSecurityReviewer) Read(filed ...string) (ok bool) { + o := orm.NewOrm() + err := o.Read(s, filed...) + if err != nil { + logs.Error(err) + return false + } + return true } \ No newline at end of file diff --git a/cve-vulner-manager/models/uploadcve.go b/cve-vulner-manager/models/uploadcve.go index af55dda..c870a53 100644 --- a/cve-vulner-manager/models/uploadcve.go +++ b/cve-vulner-manager/models/uploadcve.go @@ -1109,3 +1109,66 @@ func DeleteOriginCveRecord(beforeDate string) { err := o.Raw("delete from cve_origin_upstream_record where create_time < ?", beforeDate).QueryRow() logs.Info("DeleteOriginCveRecord", err) } + +func GetOpenLookengYaml(opy *OpenLookengYaml, colName ...string) error { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("OpenLooKeng request query parameters: ", *opy) + return errors.New("OpenLooKeng request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { + err := o.Raw("select * from cve_open_lookeng_yaml where package_name = ? and version = ? "+ + "order by id asc limit ?", opy.PackageName, opy.Version, 1).QueryRow(opy) + if err != nil { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetOpenLookengYaml1, err: ", err, *opy) + } + return err + } else { + err := o.Raw("select * from cve_open_lookeng_yaml where package_name = ? "+ + "order by id asc limit ?", opy.PackageName, 1).QueryRow(opy) + if err != nil { + logs.Error("GetOpenLookengYaml12, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return err + } +} + +func GetOpenLookengYamlAll(opy *OpenLookengYaml) (msy []OpenLookengYaml, err error) { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("GetOpenLookengYamlAll, openLookeng request query parameters: ", *opy) + return msy, errors.New("openLookeng request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { + verStr := "(" + verList := strings.Split(opy.Version, ",") + for _, ver := range verList { + verStr += "'" + ver + "'," + } + verStr = verStr[:len(verStr)-1] + ")" + sql := fmt.Sprintf(`select * from cve_open_lookeng_yaml where package_name = '%s' +and version in %s group by repo_name order by id asc`, opy.PackageName, verStr) + num, err := o.Raw(sql).QueryRows(&msy) + if num == 0 { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetOpenLookengYamlAll, err: ", err, *opy) + } + return msy, err + } else { + num, err := o.Raw("select * from cve_open_lookeng_yaml where package_name = ? "+ + "order by id asc", opy.PackageName).QueryRows(&msy) + if num == 0 { + logs.Error("GetOpenLookengYamlAll2, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return msy, err + } +} diff --git a/cve-vulner-manager/task/issue.go b/cve-vulner-manager/task/issue.go index 307ff4f..4292f12 100644 --- a/cve-vulner-manager/task/issue.go +++ b/cve-vulner-manager/task/issue.go @@ -6,6 +6,8 @@ import ( "cvevulner/taskhandler" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" + "strconv" + "strings" ) // Verify whether the issue on gitee has been deleted @@ -24,11 +26,39 @@ func CheckIssueIsExist() error { // Statistics of issues created in different communities, uncreated issues are created func CommunityIssueStatist() error { // Get the configuration file + yamlConfigTable := beego.AppConfig.String("cve::yaml_config_table") + tableList := strings.Split(yamlConfigTable, ",") + localList := make([][]string, 0) cyc := models.GetCommunityYamlConfigAll() if len(cyc) > 0 { for _, c := range cyc { taskhandler.StatistCommunityIssue(c) } } + for _, tab := range tableList { + flag := false + tabList := strings.Split(tab, ":") + for _, c := range cyc { + if tabList[0] == c.TableName { + flag = true + break + } + } + if !flag { + if len(tabList) >1 { + localList = append(localList, tabList) + } + } + } + if len(localList) > 0 { + for _, lc := range localList { + organizationID, _ := strconv.Atoi(lc[1]) + cyc := models.CommunityYamlConfig{TableName: lc[0], OrganizationID: int8(organizationID)} + num, err := models.InsertCommunityYamlConfig(&cyc) + if err != nil { + logs.Error("num: ", num, ",err: ", err) + } + } + } return nil } diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go index dc052c4..663af87 100644 --- a/cve-vulner-manager/task/issuetask.go +++ b/cve-vulner-manager/task/issuetask.go @@ -169,25 +169,7 @@ func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, fl "err: ", lockErr, ",data: ", issueValue) continue } - owner := beego.AppConfig.String("gitee::owner") - accessToken := beego.AppConfig.String("gitee::git_token") - if issueValue.OrganizationID == 3 { - owner = beego.AppConfig.String("mindspore::mindspore_owner") - accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") - } else if issueValue.OrganizationID == 2 { - owner = beego.AppConfig.String("opengauss::gauss_owner") - accessToken = beego.AppConfig.String("opengauss::git_gauss_token") - } else { - se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} - seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") - if seError == nil && se.Id > 0 { - models.UpdateIssueStatus(issueValue, 5) - logs.Info("addUnlimitedIssue, The current issue does not need to be processed, "+ - "it has been processed, cveData: ", issueValue) - ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) - continue - } - } + owner, accessToken := common.GetOwnerAndToken(issueValue.OrganizationID) // Determine whether the issue has been processed goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName, issueValue.OrganizationID) if oks { @@ -201,6 +183,15 @@ func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, fl } } if issueValue.OrganizationID == 1 { + se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} + seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") + if seError == nil && se.Id > 0 { + models.UpdateIssueStatus(issueValue, 5) + logs.Info("addUnlimitedIssue, The current issue does not need to be processed, "+ + "it has been processed, cveData: ", issueValue) + ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) + continue + } // Determine whether cve has been processed issueExist, _ := taskhandler.GetCveSecurityNotice(issueValue.CveNum, issueValue.PackName) if issueExist { @@ -317,25 +308,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i " process the next data, err: ", lockErr, ",data: ", issueValue) continue } - owner := beego.AppConfig.String("gitee::owner") - accessToken := beego.AppConfig.String("gitee::git_token") - if issueValue.OrganizationID == 3 { - owner = beego.AppConfig.String("mindspore::mindspore_owner") - accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") - } else if issueValue.OrganizationID == 2 { - owner = beego.AppConfig.String("opengauss::gauss_owner") - accessToken = beego.AppConfig.String("opengauss::git_gauss_token") - } - se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, - PackName: issueValue.PackName, Status: 1} - seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") - if seError == nil && se.Id > 0 { - models.UpdateIssueStatus(issueValue, 5) - logs.Info("addLimitedIssue, The current issue does not need to be processed,"+ - " it has been processed, cveData: ", issueValue) - ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) - continue - } + owner, accessToken := common.GetOwnerAndToken(issueValue.OrganizationID) goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName, issueValue.OrganizationID) if oks { if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || @@ -349,6 +322,16 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i } // Determine whether the issue has been processed if issueValue.OrganizationID == 1 { + se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, + PackName: issueValue.PackName, Status: 1} + seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") + if seError == nil && se.Id > 0 { + models.UpdateIssueStatus(issueValue, 5) + logs.Info("addLimitedIssue, The current issue does not need to be processed,"+ + " it has been processed, cveData: ", issueValue) + ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) + continue + } // Determine whether cve has been processed issueExist, _ := taskhandler.GetCveSecurityNotice(issueValue.CveNum, issueValue.PackName) if issueExist { @@ -397,6 +380,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i } // Process each piece of cve data if issueValue.Status == 0 { + issueValue.Status = 2 mutex.Lock() err := ProcIssue(issueValue, accessToken, owner) mutex.Unlock() @@ -408,6 +392,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i continue } } else { + issueValue.Status = 2 mutex.Lock() err := ProcUpdateIssue(issueValue, accessToken, owner) mutex.Unlock() @@ -765,6 +750,45 @@ func MindSporeIssue(issueValue models.VulnCenter, accessToken, owner, path, assi return nil } +func OpenLooKengIssue(issueValue models.VulnCenter, accessToken, owner, path, assignee string) error { + reviewerList, rlerr := models.GetOpenLookengSecurityReviewerList() + if len(reviewerList) > 0 { + for _, v := range reviewerList { + assignee = v.NameSpace + break + } + } else { + logs.Error("GetSporeSecurityReviewerList, rlerr: ", rlerr) + } + branchList := make([]string, 0) + // Get branch information + branchList = taskhandler.CreateBrandAndTags(accessToken, owner, path, issueValue.OrganizationID) + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil { + logs.Error("ProcIssue, Failed to get Score, err: ", err, ", cveId: ", issueValue.CveId) + return err + } + // Judgment of necessary fields + if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + logs.Error("ProcIssue, Field is empty: NVDScore: ", sc.NVDScore, ",Description: ", + issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", + issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) + // Update issue status + models.UpdateIssueStatus(issueValue, 6) + return errors.New("Field is empty") + } + resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, branchList) + if err != nil && err.Error() != "Recreate issue" { + logs.Error("ProcIssue, Failed to create issue, err: ", err, + ",resp: ", resp, ",cveId: ", issueValue.CveId) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) + return err + } + return nil +} + //ProcIssue Deal with issue func ProcIssue(issueValue models.VulnCenter, accessToken, owner string) error { assignee := "" @@ -781,12 +805,18 @@ func ProcIssue(issueValue models.VulnCenter, accessToken, owner string) error { logs.Error("GaussIssue, gaussErr: ", gaussErr, issueValue.CveNum) return gaussErr } - } else { + } else if issueValue.OrganizationID == 3 { sporeErr := MindSporeIssue(issueValue, accessToken, owner, path, assignee) if sporeErr != nil { logs.Error("MindSporeIssue, sporeErr: ", sporeErr, issueValue.CveNum) return sporeErr } + } else if issueValue.OrganizationID == 4 { + looKengErr := OpenLooKengIssue(issueValue, accessToken, owner, path, assignee) + if looKengErr != nil { + logs.Error("OpenLooKengIssue, looKengErr: ", looKengErr, issueValue.CveNum) + return looKengErr + } } return nil } diff --git a/cve-vulner-manager/taskhandler/assist.go b/cve-vulner-manager/taskhandler/assist.go index 612d8b7..43263e7 100644 --- a/cve-vulner-manager/taskhandler/assist.go +++ b/cve-vulner-manager/taskhandler/assist.go @@ -1,12 +1,14 @@ package taskhandler import ( + "cvevulner/common" "cvevulner/util" "encoding/json" "errors" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" "os" + "sort" "strings" ) @@ -80,7 +82,12 @@ func GetCollaboratorInfo(accessToken string, owner string, path string) (string, func GetBranchesInfo(accessToken, owner, path string, organizationID int8) ([]string, error) { BrandVerSlice := make([]string, 0) - if organizationID == 3 { + if organizationID == 4 { + openlookengVersion := beego.AppConfig.String("openlookeng::openlookeng_version") + if len(openlookengVersion) > 0 { + BrandVerSlice = strings.Split(openlookengVersion, ",") + } + } else if organizationID == 3 { mindsporeVersion := beego.AppConfig.String("mindspore::mindspore_version") if len(mindsporeVersion) > 0 { BrandVerSlice = strings.Split(mindsporeVersion, ",") @@ -109,19 +116,67 @@ func GetBranchesInfo(accessToken, owner, path string, organizationID int8) ([]st } mapValue := value["name"].(string) if mapValue != "" && len(mapValue) > 1 { - for _, gv := range BrandVerSlice { - if gv == mapValue { + if organizationID == 2 { + if !common.IsContainsLetters(mapValue) { + branchNameList = append(branchNameList, mapValue) + } + } else if organizationID == 3 { + if common.IsContainSpecialChar(mapValue, 1) { branchNameList = append(branchNameList, mapValue) } + } else if organizationID == 4 { + if !common.IsContainsLetters(mapValue) { + branchNameList = append(branchNameList, mapValue) + } + } else { + for _, gv := range BrandVerSlice { + if gv == mapValue { + branchNameList = append(branchNameList, mapValue) + } + } } } } } else { logs.Error("Failed to get branch name, "+ "err: ", err, ",owner: ", owner, ",path: ", path) + if organizationID != 1 { + logs.Info("1.Obtained branch information: ", BrandVerSlice) + return BrandVerSlice, err + } + logs.Info("2.Obtained branch information: ", branchNameList) return branchNameList, err } } + if len(branchNameList) > 0 { + sort.Strings(branchNameList) + } + if organizationID == 2 { + if len(branchNameList) > 2 { + branchNameList = branchNameList[len(branchNameList)-2:] + } + } else if organizationID == 3 { + if len(branchNameList) > 4 { + branchNameList = branchNameList[len(branchNameList)-4:] + } + } else if organizationID == 4 { + if len(branchNameList) > 1 { + branchNameList = branchNameList[len(branchNameList)-1:] + } + } + if len(BrandVerSlice) > 0 { + branchNameList = append(branchNameList, BrandVerSlice...) + } + repList := make([]string, 0) + if len(branchNameList) > 0 { + sort.Strings(branchNameList) + tmpList := (Duplicate(branchNameList)) + for _, tl := range tmpList { + repList = append(repList, tl.(string)) + } + branchNameList = repList + } + logs.Info("3.Obtained branch information: ", branchNameList) return branchNameList, nil } @@ -133,12 +188,20 @@ func GetTagsInfo(accessToken, owner, path string, organizationID int8) ([]string if err == nil && branch != nil { for _, value := range branch { if _, ok := value["name"]; !ok { - logs.Error("branches, err: ", ok, "url: ", url) + logs.Error("tagsInfo, err: ", ok, "url: ", url) continue } mapValue := value["name"].(string) if mapValue != "" && len(mapValue) > 1 { - tagsList = append(tagsList, mapValue) + if organizationID == 4 { + if !common.IsContainsLetters(mapValue) { + tagsList = append(tagsList, mapValue) + } + } else { + if common.IsContainSpecialChar(mapValue, 2) { + tagsList = append(tagsList, mapValue) + } + } } } } else { @@ -147,5 +210,18 @@ func GetTagsInfo(accessToken, owner, path string, organizationID int8) ([]string return tagsList, err } } + if len(tagsList) > 0 { + sort.Strings(tagsList) + } + if organizationID == 3 { + if len(tagsList) > 4 { + tagsList = tagsList[len(tagsList)-4:] + } + } else if organizationID == 4 { + if len(tagsList) > 1 { + tagsList = tagsList[len(tagsList)-1:] + } + } + logs.Info("GetTagsInfo, tagsList:", tagsList) return tagsList, nil } diff --git a/cve-vulner-manager/taskhandler/common.go b/cve-vulner-manager/taskhandler/common.go index 0b6e494..c87e7bc 100644 --- a/cve-vulner-manager/taskhandler/common.go +++ b/cve-vulner-manager/taskhandler/common.go @@ -243,6 +243,53 @@ const SporeBodyUpTpl = `一、漏洞信息 受影响版本排查(受影响/不受影响)： %v ` +const LooKengBodyTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openLooKeng评分： + %v + 受影响版本排查(受影响/不受影响)： + %v +` +const LooKengBodyUpTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞创建时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openLooKeng评分： + %v + Vector：CVSS：%v + 受影响版本排查(受影响/不受影响)： + %v +` const gaussCommentCopyValue = ` %v @@ -279,6 +326,27 @@ const SporeCommentCopyValue = ` MindSpore评分: (评分和向量) +受影响版本排查(受影响/不受影响): +%v +----------------------------------------------------------------------- +issue处理具体操作请参考: +%v +pr关联issue具体操作请参考: +%v +` +const LooKengCommentCopyValue = ` +%v +**issue处理注意事项:** +**1. 当前issue受影响的分支提交pr时, 须在pr描述中填写当前issue编号进行关联, 否则无法关闭当前issue;** +**2. 模板内容需要填写完整, 无论是受影响或者不受影响都需要填写完整内容,未引入的分支不需要填写, 否则无法关闭当前issue;** +**3. 以下为模板中需要填写完整的内容, 请复制到评论区回复, 注: 内容的标题名称(影响性分析说明, openLooKeng评分, 受影响版本排查(受影响/不受影响))不能省略,省略后cve-manager将无法正常解析填写内容.** +************************************************************************ +影响性分析说明: + + +openLooKeng评分: (评分和向量) + + 受影响版本排查(受影响/不受影响): %v ----------------------------------------------------------------------- @@ -353,6 +421,25 @@ func GaussCommentTemplate(assignee, commentCmd, affectedVersion string) string { return commentTemplate } +func LooKengCommentTemplate(assignee, commentCmd, affectedVersion string) string { + anName := []string{} + reviewerList, rlerr := models.GetOpenLookengSecurityReviewerList() + if len(reviewerList) > 0 { + for _, v := range reviewerList { + anName = append(anName, "@"+v.NameSpace+" ") + } + } else { + logs.Error("GetOpenLookengSecurityReviewerList, rlerr: ", rlerr) + } + if len(anName) > 1 { + assignee = strings.Join(anName, ",") + } else { + assignee = "@" + assignee + " " + } + commentTemplate := fmt.Sprintf(LooKengCommentCopyValue, assignee, affectedVersion, commentCmd, PrIssueLink) + return commentTemplate +} + func GitOpenEulerData(values map[string]interface{}, ge *models.GitOpenEuler, tb models.GitPackageTable) { defer common.Catchs() ge.TableName = tb.TableName @@ -622,7 +709,7 @@ func AffectVersionExtract(brandArray []string, itsAffectedVersion, packName stri } else { brandx = brand } - logs.Info("brand1: ", brandx) + //logs.Info("brand1: ", brandx) affectedVersion = affectedVersion + strconv.Itoa(i+1) + "." + brandx + "\n" } } @@ -646,6 +733,7 @@ func AffectVersionExtract(brandArray []string, itsAffectedVersion, packName stri affectedVersion = "\n" } } + logs.Info("affectedVersion: ", affectedVersion) return affectedVersion } @@ -717,7 +805,11 @@ func CreateIssueBody(accessToken, owner, path, assignee string, scoreType = "3.0" } if len(brandArray) == 0 { - brandArray, _ = GetBranchesInfo(accessToken, owner, path, cve.OrganizationID) + if cve.OrganizationID == 1 || cve.OrganizationID == 2 { + brandArray, _ = GetBranchesInfo(accessToken, owner, path, cve.OrganizationID) + } else if cve.OrganizationID == 3 || cve.OrganizationID == 4 { + brandArray = CreateBrandAndTags(accessToken, owner, path, cve.OrganizationID) + } } if cve.Description != "" && len(cve.Description) > 1 { cve.Description = strings.Replace(cve.Description, "\n", "", -1) @@ -733,6 +825,7 @@ func CreateIssueBody(accessToken, owner, path, assignee string, commentCmd := BConfig.String("reflink::comment_cmd") gaussCommentCmd := BConfig.String("reflink::gauss_comment_cmd") sporeCommentCmd := BConfig.String("reflink::spore_comment_cmd") + looKengCommentCmd := BConfig.String("reflink::looKeng_comment_cmd") floatOpenEulerScore, err := strconv.ParseFloat(openEulerScore, 64) if err != nil { floatOpenEulerScore = 0.0 @@ -807,6 +900,10 @@ func CreateIssueBody(accessToken, owner, path, assignee string, bodyUpTplx = SporeBodyUpTpl bodyTplx = SporeBodyTpl commentCmd = sporeCommentCmd + } else if cve.OrganizationID == 4 { + bodyUpTplx = LooKengBodyUpTpl + bodyTplx = LooKengBodyTpl + commentCmd = looKengCommentCmd } cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" cve.CveDetailUrl = "https://nvd.nist.gov/vuln/detail/" + cve.CveNum @@ -1013,7 +1110,7 @@ func AffectVersion(affectedVersion string) int { func Duplicate(a interface{}) (ret []interface{}) { va := reflect.ValueOf(a) for i := 0; i < va.Len(); i++ { - if i > 0 && reflect.DeepEqual(va.Index(i-1).Interface(), va.Index(i).Interface()) { + if i > 0 && reflect.DeepEqual(va.Index(i - 1).Interface(), va.Index(i).Interface()) { continue } ret = append(ret, va.Index(i).Interface()) diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index 7cd7428..4f1e370 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -10,6 +10,7 @@ import ( "github.com/astaxie/beego" "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" + "sort" "strconv" "strings" "sync" @@ -448,7 +449,7 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, return "", nil } -func UpdateIssueToGit(accessToken string, owner string, path string, +func UpdateIssueToGit(accessToken, owner, path string, cve models.VulnCenter, its models.IssueTemplate) (string, error) { logs.Info("UpdateIssueToGit, Update template request parameters: cve: ", cve, ",its: ", its, ", owner: ", owner, ",path: ", path) @@ -458,15 +459,11 @@ func UpdateIssueToGit(accessToken string, owner string, path string, models.UpdateIssueStatus(cve, 2) return "", errors.New("The current issue has been suspended/rejected and will not be processed") } + owner, accessToken = common.GetOwnerAndToken(cve.OrganizationID) if cve.OrganizationID == 2 { - owner = beego.AppConfig.String("opengauss::gauss_owner") - accessToken = beego.AppConfig.String("opengauss::git_gauss_token") if len(path) < 2 { path = beego.AppConfig.String("opengauss::gauss_issue_path") } - } else if cve.OrganizationID == 3 { - owner = beego.AppConfig.String("mindspore::mindspore_owner") - accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") } assigneeGite := "" if cve.DataSource == 4 && cve.CveId == its.CveId && len(its.IssueNum) > 2 { @@ -770,7 +767,10 @@ func CreateIssueComment(accessToken, owner, path, assignee string, } commentCmd := BConfig.String("reflink::comment_cmd") commentBody := "" - if cve.OrganizationID == 3 { + if cve.OrganizationID == 4 { + commentCmd = BConfig.String("reflink::looKeng_comment_cmd") + commentBody = LooKengCommentTemplate(assignee, commentCmd, affectedVersion) + } else if cve.OrganizationID == 3 { commentCmd = BConfig.String("reflink::spore_comment_cmd") commentBody = SporeCommentTemplate(assignee, commentCmd, affectedVersion) } else if cve.OrganizationID == 2 { @@ -844,7 +844,12 @@ func CreateSecNoticeData(sec *models.SecurityNotice, iss models.VulnCenter, iss.RepoName = iss.PackName } sec.Introduction = "An update for " + iss.RepoName + " is now available for " + branchs + "." - if iss.OrganizationID == 3 { + if iss.OrganizationID == 4 { + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openLooKeng Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + + " Scoring System(CVSS)base score,which gives a detailed severity rating," + + " is available for each vulnerability from the CVElink(s) in the References section." + } else if iss.OrganizationID == 3 { sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "MindSpore Security has rated this" + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + " Scoring System(CVSS)base score,which gives a detailed severity rating," + @@ -869,46 +874,54 @@ func CreateBrandAndTags(accessToken, owner, path string, organizationID int8) [] if branchList == nil || len(branchList) == 0 { logs.Error("ProcIssue, Failed to obtain branch information,CveNum: ", "", ", path: ", path, ", err: ", errBrands) - sporeVersion := beego.AppConfig.String("mindspore::mindspore_version") - if len(sporeVersion) > 0 { - sporeVerSlice := strings.Split(sporeVersion, ",") - if len(sporeVerSlice) > 0 { - branchList = append(branchList, sporeVerSlice...) - } - } } if len(branchList) > 0 { tagList, tagErr := GetTagsInfo(accessToken, owner, path, organizationID) if tagList == nil || len(tagList) == 0 { - logs.Error("GetTagsInfo, Failed to obtain tag information,CveNum: ", "", - ", path: ", path, ", err: ", tagErr) - branchList = []string{} - } - for _, branch := range branchList { - // delete data - models.DelMindSporeBrandTagsByBrand(branch, path) - bl := common.StripStrReg(branch) - for _, tags := range tagList { - if strings.Contains(tags, bl) { - mbt := models.MindSporeBrandTags{PackageName: path, Brand: branch, Tags: tags, CreateTime: common.GetCurTime()} - spId, spErr := models.InsertMindSporeBrandTags(&mbt) - if spErr != nil { - logs.Error(spId, spErr) - continue + logs.Error("GetTagsInfo, Failed to obtain tag information, path: ", path, ",owner:", owner, ", err: ", tagErr) + tagsList = append(tagsList, "master") + } else { + if organizationID == 3 { + for _, branch := range branchList { + // delete data + models.DelMindSporeBrandTagsByBrand(branch, path) + bl := common.StripStrReg(branch) + for _, tags := range tagList { + if strings.Contains(tags, bl) { + mbt := models.MindSporeBrandTags{PackageName: path, Brand: branch, Tags: tags, CreateTime: common.GetCurTime()} + spId, spErr := models.InsertMindSporeBrandTags(&mbt) + if spErr != nil { + logs.Error(spId, spErr) + continue + } + tagsList = append(tagsList, tags) + } + } + if branch == "master" { + mbt := models.MindSporeBrandTags{PackageName: path, Brand: branch, Tags: branch, CreateTime: common.GetCurTime()} + spId, spErr := models.InsertMindSporeBrandTags(&mbt) + if spErr != nil { + logs.Error(spId, spErr) + continue + } + tagsList = append(tagsList, branch) } - tagsList = append(tagsList, tags) - } - } - if branch == "master" { - mbt := models.MindSporeBrandTags{PackageName: path, Brand: branch, Tags: branch, CreateTime: common.GetCurTime()} - spId, spErr := models.InsertMindSporeBrandTags(&mbt) - if spErr != nil { - logs.Error(spId, spErr) - continue } - tagsList = append(tagsList, branch) + } else if organizationID == 4 { + tagsList = append(tagsList, tagList...) + tagsList = append(tagsList, "master") } } } + repList := make([]string, 0) + if len(tagsList) > 0 { + sort.Strings(tagsList) + tmpList := (Duplicate(tagsList)) + for _, tl := range tmpList { + repList = append(repList, tl.(string)) + } + tagsList = repList + } + logs.Info("CreateBrandAndTags, tagsList: ", tagsList) return tagsList } diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index 1e90203..b8801f2 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -41,6 +41,11 @@ type UpdateInfoXml struct { AffectBranch string } +type PackageVersion struct { + OrganizationID int8 + VerionList []string +} + func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNum int, CveRes models.VulnCenter, goe models.GitPackageInfo, organizationID int8) (bool, error) { @@ -850,13 +855,16 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) if key != "" && len(key) > 1 { pkList := []string{} pkList = append(pkList, key) - versionFlag := false + eulerFlag := false gaussFlag := false sporeFlag := false + looKengFlag := false var goe models.GitPackageInfo + pvList := make([]PackageVersion, 0) openGausVersion := make([]string, 0) mindSporeVersion := make([]string, 0) openEulerVersion := make([]string, 0) + openLooKengVersion := make([]string, 0) versionList := make([]string, 0) if value != "" && len(value) > 0 { versionList = strings.Split(value, ",") @@ -875,9 +883,15 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) mindSporeVersion = append(mindSporeVersion, ver) sporeFlag = true } + olky := models.OpenLookengYaml{PackageName: key, Version: ver} + looKengErr := models.GetOpenLookengYaml(&olky, "PackageName", "Version") + if looKengErr == nil && olky.Id > 0 { + openLooKengVersion = append(openLooKengVersion, ver) + looKengFlag = true + } goe, ok = models.QueryCveOpeneulerDetaildataByName(key, ver) if ok { - versionFlag = true + eulerFlag = true openEulerVersion = append(openEulerVersion, ver) } else { logs.Error("GenCveVuler, The corresponding warehouse data is not queried, data: ", cveData) @@ -886,44 +900,47 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) } } } - if !versionFlag && !gaussFlag && !sporeFlag { + if !eulerFlag && !gaussFlag && !sporeFlag && !looKengFlag { models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 4) logs.Error("GenCveVuler, The version information corresponds to the error, ", cveData.PackName, cveData.Version, cveData.CveNum) UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "The cve component or version information cannot be matched to the corresponding data", 4) - return false, errors.New("数据错误，暂时不处理") + continue } - organizationList := make([]int8, 0) organizationID := int8(1) - if versionFlag { - organizationList = append(organizationList, organizationID) + if eulerFlag { + pv := PackageVersion{OrganizationID: organizationID, VerionList: openEulerVersion} + pvList = append(pvList, pv) } if gaussFlag { organizationID = int8(2) - organizationList = append(organizationList, organizationID) + pv := PackageVersion{OrganizationID: organizationID, VerionList: openGausVersion} + pvList = append(pvList, pv) } if sporeFlag { organizationID = int8(3) - organizationList = append(organizationList, organizationID) + pv := PackageVersion{OrganizationID: organizationID, VerionList: mindSporeVersion} + pvList = append(pvList, pv) + } + if looKengFlag { + organizationID = int8(4) + pv := PackageVersion{OrganizationID: organizationID, VerionList: openLooKengVersion} + pvList = append(pvList, pv) } failFlag := false - for _, orId := range organizationList { - ok, addErr := AddOrSelectToCenter(key, cveRef, scopeType, value, cveData, - openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulernum, - cveDesc, cveScV3, goe, cveScV2, orId) - if !ok || addErr != nil { - logs.Error("AddOrSelectToCenter, addErr: ", addErr) - failFlag = true - continue - } + ok, addErr := AddOrSelectToCenter(key, cveRef, scopeType, value, cveData, + pvList, pkList, openeulernum, cveDesc, cveScV3, goe, cveScV2) + if !ok || addErr != nil { + logs.Error("AddOrSelectToCenter, addErr: ", addErr) + failFlag = true } if failFlag { logs.Error("Part of the data failed") models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "Version does not match, data synchronization failed", 3) - return false, errors.New("Part of the data failed") + continue } } } @@ -933,84 +950,117 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) } func AddOrSelectToCenter(packageName, cveRef, scopeType, value string, cveData models.OriginUpstream, - openGausVersion, openEulerVersion, mindSporeVersion, pkList []string, openeulerNum int, + pvList []PackageVersion, pkList []string, openeulerNum int, cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, - cveScV2 models.OriginUpstreamImpactScoreV2, organizationID int8) (bool, error) { - if organizationID == 3 { - if len(mindSporeVersion) > 0 { - mindMap := make(map[string]string) - for _, ver := range mindSporeVersion { - ms := models.MindSporeYaml{PackageName: packageName, Version: ver} - msy, mErr := models.GetMindSporeYamlAll(&ms) - if len(msy) > 0 { - for _, my := range msy { - if _, ok := mindMap[my.Repo]; !ok { - mindMap[my.Repo] = packageName + cveScV2 models.OriginUpstreamImpactScoreV2) (bool, error) { + for _, pv := range pvList { + organizationID := pv.OrganizationID + if organizationID == 4 { + looKengVersion := pv.VerionList + if len(looKengVersion) > 0 { + looKengMap := make(map[string]string) + for _, ver := range looKengVersion { + olky := models.OpenLookengYaml{PackageName: packageName, Version: ver} + msy, mErr := models.GetOpenLookengYamlAll(&olky) + if len(msy) > 0 { + for _, my := range msy { + if _, ok := looKengMap[my.Repo]; !ok { + looKengMap[my.Repo] = packageName + } } + } else { + logs.Error("openLooKeng, mErr: ", mErr) } - } else { - logs.Error("MindSpore, mErr: ", mErr) } - } - if len(mindMap) > 0 { - for repo, _ := range mindMap { - ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, - openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, - cveDesc, cveScV3, goe, cveScV2, organizationID) - if !ok { - logs.Error("MindSpore, dErr: ", dErr) + if len(looKengMap) > 0 { + for repo, _ := range looKengMap { + ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, + looKengVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, pv.OrganizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) + } } } } - } - } else if organizationID == 2 { - if len(openGausVersion) > 0 { - gaussMap := make(map[string]string) - for _, ver := range openGausVersion { - opy := models.OpenGussYaml{PackageName: packageName, Version: ver} - ogy, oErr := models.GetOpengaussYamlAll(&opy) - if len(ogy) > 0 { - for _, gy := range ogy { - if _, ok := gaussMap[gy.Repo]; !ok { - gaussMap[gy.Repo] = packageName + } else if organizationID == 3 { + mindSporeVersion := pv.VerionList + if len(mindSporeVersion) > 0 { + mindMap := make(map[string]string) + for _, ver := range mindSporeVersion { + ms := models.MindSporeYaml{PackageName: packageName, Version: ver} + msy, mErr := models.GetMindSporeYamlAll(&ms) + if len(msy) > 0 { + for _, my := range msy { + if _, ok := mindMap[my.Repo]; !ok { + mindMap[my.Repo] = packageName + } + } + } else { + logs.Error("MindSpore, mErr: ", mErr) + } + } + if len(mindMap) > 0 { + for repo, _ := range mindMap { + ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, + mindSporeVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) } } - } else { - logs.Error("openGauss, dErr: ", oErr) } } - if len(gaussMap) > 0 { - for repo, _ := range gaussMap { - ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, - openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, - cveDesc, cveScV3, goe, cveScV2, organizationID) - if !ok { - logs.Error("openGauss, dErr: ", dErr) + } else if organizationID == 2 { + openGausVersion := pv.VerionList + if len(openGausVersion) > 0 { + gaussMap := make(map[string]string) + for _, ver := range openGausVersion { + opy := models.OpenGussYaml{PackageName: packageName, Version: ver} + ogy, oErr := models.GetOpengaussYamlAll(&opy) + if len(ogy) > 0 { + for _, gy := range ogy { + if _, ok := gaussMap[gy.Repo]; !ok { + gaussMap[gy.Repo] = packageName + } + } + } else { + logs.Error("openGauss, dErr: ", oErr) + } + } + if len(gaussMap) > 0 { + for repo, _ := range gaussMap { + ok, dErr := AddOrDataToCenter(repo, packageName, cveRef, scopeType, value, cveData, + openGausVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("openGauss, dErr: ", dErr) + } } } } - } - } else { - ok, dErr := AddOrDataToCenter(packageName, packageName, cveRef, scopeType, value, cveData, - openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, - cveDesc, cveScV3, goe, cveScV2, organizationID) - if !ok { - logs.Error("openEuler, dErr: ", dErr) - return false, dErr + } else { + openEulerVersion := pv.VerionList + ok, dErr := AddOrDataToCenter(packageName, packageName, cveRef, scopeType, value, cveData, + openEulerVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("openEuler, dErr: ", dErr) + return false, dErr + } } } return true, nil } func AddOrDataToCenter(repoNme, packageName, cveRef, scopeType, value string, cveData models.OriginUpstream, - openGausVersion, openEulerVersion, mindSporeVersion, pkList []string, openeulerNum int, + versionList, pkList []string, openeulerNum int, cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, cveScV2 models.OriginUpstreamImpactScoreV2, organizationID int8) (bool, error) { CveRes, err := models.QueryCveByNum(cveData.CveNum, repoNme, organizationID) if err { CveRes.RepoName = packageName - retVersion := AddCveVersion(organizationID, openGausVersion, - openEulerVersion, mindSporeVersion, CveRes.CveVersion) + retVersion := AddCveVersion(organizationID, versionList, CveRes.CveVersion) pkList = append(pkList, retVersion) lockx.Lock() ok, err := UpdateCveGroups(cveData, cveRef, openeulerNum, CveRes, cveDesc, cveScV3, goe, @@ -1022,8 +1072,7 @@ func AddOrDataToCenter(repoNme, packageName, cveRef, scopeType, value string, cv return false, errors.New("数据错误，暂时不处理") } } else { - retVersion := AddCveVersion(organizationID, openGausVersion, - openEulerVersion, mindSporeVersion, "") + retVersion := AddCveVersion(organizationID, versionList, "") pkList = append(pkList, retVersion) lockx.Lock() ok, err := InsertCveGroups(cveData, cveRef, repoNme, openeulerNum, cveDesc, cveScV3, goe, @@ -1053,52 +1102,37 @@ func UpdateOriginUpstreamRecord(ids, cveNum, errDesc string, errStatus int8) { } } -func AddCveVersion(organizationID int8, openGausVersion, - openEulerVersion, mindSporeVersion []string, cveVersion string) (retVersion string) { +func AddCveVersion(organizationID int8, versionList []string, cveVersion string) (retVersion string) { retVersion = cveVersion - cveVerList := strings.Split(cveVersion, ",") - if organizationID == 3 { - if len(cveVerList) > 0 && len(mindSporeVersion) > 0 { - mindSporeVersion = append(mindSporeVersion, cveVerList...) - } - mindSlice := make([]string, 0) - if len(mindSporeVersion) > 0 { - sort.Strings(mindSporeVersion) - repSlice := common.SliceRemoveDup(mindSporeVersion) + cveVerList := []string{} + if len(cveVersion) > 0 { + cveVerList = strings.Split(cveVersion, ",") + } + if organizationID != 1 { + if len(cveVerList) > 0 && len(versionList) > 0 { + versionList = append(versionList, cveVerList...) + } + verSlice := make([]string, 0) + if len(versionList) > 0 { + sort.Strings(versionList) + repSlice := common.SliceRemoveDup(versionList) for _, rep := range repSlice { if len(rep.(string)) > 0 { - mindSlice = append(mindSlice, rep.(string)) + verSlice = append(verSlice, rep.(string)) } } } - if len(mindSlice) > 0 { - retVersion = strings.Join(mindSlice, ",") - } - } else if organizationID == 2 { - if len(cveVerList) > 0 && len(openGausVersion) > 0 { - openGausVersion = append(openGausVersion, cveVerList...) - } - gaussSlice := make([]string, 0) - if len(openGausVersion) > 0 { - sort.Strings(openGausVersion) - repSlice := common.SliceRemoveDup(openGausVersion) - for _, rep := range repSlice { - if len(rep.(string)) > 0 { - gaussSlice = append(gaussSlice, rep.(string)) - } - } - } - if len(gaussSlice) > 0 { - retVersion = strings.Join(gaussSlice, ",") + if len(verSlice) > 0 { + retVersion = strings.Join(verSlice, ",") } } else { - if len(cveVerList) > 0 && len(openEulerVersion) > 0 { - openEulerVersion = append(openEulerVersion, cveVerList...) + if len(cveVerList) > 0 && len(versionList) > 0 { + versionList = append(versionList, cveVerList...) } eulerSlice := make([]string, 0) - if len(openEulerVersion) > 0 { - tmpEulerSlice := make([]string, len(openEulerVersion)) - for _, es := range openEulerVersion { + if len(versionList) > 0 { + tmpEulerSlice := make([]string, len(versionList)) + for _, es := range versionList { if len(es) > 0 { versx := common.BranchVersionRep(es) tmpEulerSlice = append(tmpEulerSlice, versx) @@ -1107,7 +1141,9 @@ func AddCveVersion(organizationID int8, openGausVersion, sort.Strings(tmpEulerSlice) repSlice := common.SliceRemoveDup(tmpEulerSlice) for _, rep := range repSlice { - eulerSlice = append(eulerSlice, rep.(string)) + if len(rep.(string)) > 0 { + eulerSlice = append(eulerSlice, rep.(string)) + } } } if len(eulerSlice) > 0 { @@ -1129,6 +1165,12 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe } organizationID := int8(1) organizationList := make([]int8, 0) + olky := models.OpenLookengYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + looKengErr := models.GetOpenLookengYaml(&olky, "PackageName", "Version") + if looKengErr == nil && olky.Id > 0 { + organizationID = 4 + organizationList = append(organizationList, organizationID) + } ms := models.MindSporeYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") if sporeErr == nil && ms.Id > 0 { @@ -1182,6 +1224,7 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe } } if failFlag { + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) return false, errors.New("数据错误，暂时不处理") } models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 2) @@ -1190,7 +1233,22 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe func AddExcelToCenter(cveData models.OriginExcel, cveRef string, openeulerNum int, goe models.GitPackageInfo, organizationID int8) (bool, error) { - if organizationID == 3 { + if organizationID == 4 { + olky := models.OpenLookengYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + opy, mErr := models.GetOpenLookengYamlAll(&olky) + if len(opy) > 0 { + for _, my := range opy { + ok, dErr := AddExDataToCenter(cveData, cveRef, my.Repo, openeulerNum, goe, organizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) + return ok, dErr + } + } + } else { + logs.Error("MindSpore, mErr: ", mErr) + return false, mErr + } + } else if organizationID == 3 { ms := models.MindSporeYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} msy, mErr := models.GetMindSporeYamlAll(&ms) if len(msy) > 0 { @@ -1236,8 +1294,7 @@ func AddExDataToCenter(cveData models.OriginExcel, CveRes, err := models.QueryCveByNum(cveData.CveNum, repoName, organizationID) if err { CveRes.RepoName = cveData.PackName - retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, - []string{cveData.CveVersion}, []string{cveData.CveVersion}, CveRes.CveVersion) + retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, CveRes.CveVersion) cveData.CveVersion = retVersion lockx.Lock() ok, err := UpdateExcelCveGroups(cveData, cveRef, openeulerNum, CveRes, goe, organizationID) @@ -1248,8 +1305,7 @@ func AddExDataToCenter(cveData models.OriginExcel, return false, errors.New("数据错误，暂时不处理") } } else { - retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, - []string{cveData.CveVersion}, []string{cveData.CveVersion}, "") + retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, "") cveData.CveVersion = retVersion lockx.Lock() ok, err := InsertCveExcelGroups(cveData, cveRef, repoName, openeulerNum, goe, organizationID) @@ -1357,17 +1413,10 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c vul.Description = briefIntroduction vul.Status = cveStatus vul.OrganizationID = cveData.OrganizationID - owner = beego.AppConfig.String("gitee::owner") - accessToken := beego.AppConfig.String("gitee::git_token") - if vul.OrganizationID == 3 { - owner = beego.AppConfig.String("mindspore::mindspore_owner") - accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") - } else if vul.OrganizationID == 2 { - owner = beego.AppConfig.String("opengauss::gauss_owner") - accessToken = beego.AppConfig.String("opengauss::git_gauss_token") - } - retVersion := AddCveVersion(vul.OrganizationID, []string{}, - strings.Split(RemoveSubstring(lop.Version, specCharList), ","), []string{}, "") + accessToken := "" + owner, accessToken = common.GetOwnerAndToken(vul.OrganizationID) + retVersion := AddCveVersion(vul.OrganizationID, + strings.Split(RemoveSubstring(lop.Version, specCharList), ","), "") vul.CveVersion = retVersion vul.RepairTime = "" vul.PackName = cveData.RepoPath @@ -1580,8 +1629,8 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c if (vul.Description == "" || len(vul.Description) < 1) && len(briefIntroduction) > 1 { vul.Description = briefIntroduction } - retVersion := AddCveVersion(cveData.OrganizationID, []string{}, - strings.Split(RemoveSubstring(lop.Version, specCharList), ","), []string{}, vul.CveVersion) + retVersion := AddCveVersion(cveData.OrganizationID, + strings.Split(RemoveSubstring(lop.Version, specCharList), ","), vul.CveVersion) vul.CveVersion = retVersion if vul.PackName == "" || len(vul.PackName) < 1 { vul.PackName = cveData.RepoPath @@ -1591,15 +1640,8 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c } vul.RepoName = lop.Components vul.OrganizationID = cveData.OrganizationID - owner = beego.AppConfig.String("gitee::owner") - accessToken := beego.AppConfig.String("gitee::git_token") - if vul.OrganizationID == 3 { - owner = beego.AppConfig.String("mindspore::mindspore_owner") - accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") - } else if vul.OrganizationID == 2 { - owner = beego.AppConfig.String("opengauss::gauss_owner") - accessToken = beego.AppConfig.String("opengauss::git_gauss_token") - } + accessToken := "" + owner, accessToken = common.GetOwnerAndToken(vul.OrganizationID) var sec models.SecurityNotice sec.CveId = vul.CveId sec.CveNum = vul.CveNum @@ -1734,7 +1776,6 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c score.Oauthentication = vectorO2.Authentication } } - } else { v2 := score.NVDScore vul.CveLevel = models.OpenEulerScoreProc(v2) diff --git a/cve-vulner-manager/taskhandler/gauss.go b/cve-vulner-manager/taskhandler/gauss.go index 7fd60af..e7fd306 100644 --- a/cve-vulner-manager/taskhandler/gauss.go +++ b/cve-vulner-manager/taskhandler/gauss.go @@ -741,8 +741,7 @@ func ParseRpmPackage(pkgList []string, gaussTempId int64, packageName, publicDat func GuassGetDateByGite(pkgList []models.GaussExcelTag, c chan<- []IssueAndPkg, startTime, affectBranch string) { defer wgTrigger.Done() - token := beego.AppConfig.String("opengauss::git_gauss_token") - owner := beego.AppConfig.String("opengauss::gauss_owner") + owner, token := common.GetOwnerAndToken(int8(2)) //gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") st := util.TimeStrToInt(startTime, "2006-01-02") chData := make([]IssueAndPkg, 0) diff --git a/cve-vulner-manager/taskhandler/hook.go b/cve-vulner-manager/taskhandler/hook.go index e9e8a06..c257395 100644 --- a/cve-vulner-manager/taskhandler/hook.go +++ b/cve-vulner-manager/taskhandler/hook.go @@ -232,34 +232,15 @@ func CheckAffectVerComplete(affectedVersion, packageName, version string, organi affectBranchsxList := make([]string, 0) affectProductList := make([]string, 0) unFixValue := make([]string, 0) - affectedBranchs := "" - if organizateId == 1 { - affectedBranchs = beego.AppConfig.String("cve::affected_branchs") - affectedBranchs = QueryEulerRepoOrigin(affectedBranchs, packageName, version, organizateId) - } else if organizateId == 2 { - affectedBranchs = beego.AppConfig.String("opengauss::gauss_version") - } else if organizateId == 3 { - affectedBranchs = beego.AppConfig.String("mindspore::mindspore_version") + owner, accessToken := common.GetOwnerAndToken(organizateId) + if organizateId == 1 || organizateId == 2 { + affectBranchsxList, _ = GetBranchesInfo(accessToken, owner, packageName, organizateId) + } else if organizateId == 3 || organizateId == 4 { + affectBranchsxList = CreateBrandAndTags(accessToken, owner, packageName, organizateId) } - if affectedBranchs != "" && len(affectedBranchs) > 0 { - affectBranchsxList = strings.Split(affectedBranchs, ",") - } else { + if len(affectBranchsxList) == 0 { return unFixValue } - if organizateId == 3 { - tmpList := make([]string, 0) - for _, brand := range affectBranchsxList { - mdbt := models.MindSporeBrandTags{PackageName: packageName, Brand: brand} - mbList := models.QueryMindSporeTagsByBrand(&mdbt) - if len(mbList) > 0 { - for _, mb := range mbList { - tmpList = append(tmpList, mb.Tags) - } - } - } - affectBranchsxList = affectBranchsxList[:0] - affectBranchsxList = append(affectBranchsxList, tmpList...) - } if affectedVersion != "" && len(affectedVersion) > 1 { affectProductList = strings.Split(affectedVersion, ",") } @@ -282,7 +263,8 @@ func CheckAffectVerComplete(affectedVersion, packageName, version string, organi affBrand := common.BranchVersionRep(branchSlice[0]) if strings.TrimSpace(affBrand) == abl { if len(strings.TrimSpace(branchSlice[1])) > 1 { - if (strings.TrimSpace(branchSlice[1]) == "受影响") || (strings.TrimSpace(branchSlice[1]) == "不受影响") { + if (strings.TrimSpace(branchSlice[1]) == "受影响") || + (strings.TrimSpace(branchSlice[1]) == "不受影响") { flag = true } } diff --git a/cve-vulner-manager/taskhandler/issue.go b/cve-vulner-manager/taskhandler/issue.go index 55d5069..3ff96a3 100644 --- a/cve-vulner-manager/taskhandler/issue.go +++ b/cve-vulner-manager/taskhandler/issue.go @@ -31,19 +31,12 @@ func ProcIssueIsExists(prnum int) error { if issueTmp.IssueNum != "" && len(issueTmp.IssueNum) > 0 { issueErr := error(nil) issueData := map[string]interface{}{} - owner := beego.AppConfig.String("gitee::owner") - accessToken := beego.AppConfig.String("gitee::git_token") path := issueTmp.Repo - if v.OrganizationID == 3 { - owner = beego.AppConfig.String("mindspore::mindspore_owner") - accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") - // Query the repo that needs to submit an issue - } else if v.OrganizationID == 2 { + owner, accessToken := common.GetOwnerAndToken(v.OrganizationID) + if v.OrganizationID == 2 { if len(path) < 2 { path = beego.AppConfig.String("opengauss::gauss_issue_path") } - owner = beego.AppConfig.String("opengauss::gauss_owner") - accessToken = beego.AppConfig.String("opengauss::git_gauss_token") } issueErr, issueData = GetGiteeIssue(accessToken, owner, path, issueTmp.IssueNum) if issueErr != nil { diff --git a/cve-vulner-manager/taskhandler/oricvecheck.go b/cve-vulner-manager/taskhandler/oricvecheck.go index f9e6d7d..1cc08fa 100644 --- a/cve-vulner-manager/taskhandler/oricvecheck.go +++ b/cve-vulner-manager/taskhandler/oricvecheck.go @@ -50,10 +50,23 @@ func CheckCveOriginData(prcnum int) (string, error) { if openErr == nil && opy.Id > 0 { models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) logs.Info("Data added to the cve vulnerability: ", cveData, ", gits: ", gits) - } else { - models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 0) - logs.Info("Correspondence between warehouse and version is wrong, restore: ", cveData, ", gits: ", gits) + continue } + ms := models.MindSporeYaml{PackageName: pkList[0], Version: pkList[1]} + sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") + if ms.Id > 0 { + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) + logs.Info("Correspondence between warehouse and version is wrong, restore: ", cveData, ", gits: ", gits, sporeErr) + continue + } + opky := models.OpenLookengYaml{PackageName: pkList[0], Version: pkList[1]} + opkyErr := models.GetOpenLookengYaml(&opky, "PackageName", "Version") + if opky.Id > 0 { + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) + logs.Info("Correspondence between warehouse and version is wrong, restore: ", cveData, ", gits: ", gits, opkyErr) + continue + } + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 0) } else { models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) logs.Info("Data added to the cve vulnerability:", cveData, ", gits: ", gits) diff --git a/cve-vulner-manager/util/parsepayload.go b/cve-vulner-manager/util/parsepayload.go index a06df64..5e47e45 100644 --- a/cve-vulner-manager/util/parsepayload.go +++ b/cve-vulner-manager/util/parsepayload.go @@ -13,6 +13,7 @@ import ( const ( KwOpenGaussScore = "openGauss评分:" KwMindSporeScore = "MindSpore评分:" + KwLooKengScore = "openLooKeng评分:" ) const ( -- Gitee From 8e595c59ef5e1f680a785527cd67b7ec286a8681 Mon Sep 17 00:00:00 2001 From: zhangjianjun Date: Thu, 23 Dec 2021 10:04:16 +0800 Subject: [PATCH 2/2] Add openLooKeng's cve code online --- .../cve-py/config_yaml/deployment.yaml | 44 ------------------- .../cve-py/config_yaml/kustomization.yaml | 17 ------- .../cve-py/config_yaml/namespace.yaml | 6 --- .../cve-py/config_yaml/secrets.yaml | 20 --------- .../cve-py/config_yaml/service.yaml | 14 ------ 5 files changed, 101 deletions(-) delete mode 100644 cve-vulner-manager/cve-py/config_yaml/deployment.yaml delete mode 100644 cve-vulner-manager/cve-py/config_yaml/kustomization.yaml delete mode 100644 cve-vulner-manager/cve-py/config_yaml/namespace.yaml delete mode 100644 cve-vulner-manager/cve-py/config_yaml/secrets.yaml delete mode 100644 cve-vulner-manager/cve-py/config_yaml/service.yaml diff --git a/cve-vulner-manager/cve-py/config_yaml/deployment.yaml b/cve-vulner-manager/cve-py/config_yaml/deployment.yaml deleted file mode 100644 index 646ce9c..0000000 --- a/cve-vulner-manager/cve-py/config_yaml/deployment.yaml +++ /dev/null @@ -1,44 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - labels: - app: cve-manager-analysis - name: cve-manager-analysis -spec: - replicas: 1 - revisionHistoryLimit: 10 - selector: - matchLabels: - app: cve-manager-analysis - template: - metadata: - labels: - app: cve-manager-analysis - spec: - containers: - - env: - - name: DB_PWDPY - valueFrom: - secretKeyRef: - name: cve-secrets-py - key: db-pwdpy - - name: DB_URI - valueFrom: - secretKeyRef: - name: cve-secrets-py - key: db-uri - - name: CVE_EMAIL_SENDADDR - valueFrom: - secretKeyRef: - name: cve-secrets-py - key: cve-email-sendaddr - - name: CVE_EMAIL_PASSWORD - valueFrom: - secretKeyRef: - name: cve-secrets-py - key: cve-email-password - - name: TZ - value: Asia/Shanghai - image: swr.cn-north-4.myhuaweicloud.com/opensourceway/openeuler/cve-manager-analysis:1cb6af4c1d428074cb8e54db23adf1efaab75639 - imagePullPolicy: IfNotPresent - name: cve-manager-analysis diff --git a/cve-vulner-manager/cve-py/config_yaml/kustomization.yaml b/cve-vulner-manager/cve-py/config_yaml/kustomization.yaml deleted file mode 100644 index 9684b1e..0000000 --- a/cve-vulner-manager/cve-py/config_yaml/kustomization.yaml +++ /dev/null @@ -1,17 +0,0 @@ -resources: -- namespace.yaml -- deployment.yaml -- service.yaml -- secrets.yaml -commonLabels: - app: cve-manager-analysis - owner: zhangjianjun -commonAnnotations: - email: 841670711@qq.com - owner: zhangjianjun -namespace: cve-manager -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -images: -- name: swr.cn-north-4.myhuaweicloud.com/opensourceway/openeuler/cve-manager-analysis - newTag: 89281957eaa42c6a7c8cd048b1374dccdf68fd96 diff --git a/cve-vulner-manager/cve-py/config_yaml/namespace.yaml b/cve-vulner-manager/cve-py/config_yaml/namespace.yaml deleted file mode 100644 index 09d5c2b..0000000 --- a/cve-vulner-manager/cve-py/config_yaml/namespace.yaml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - name: cve-manager-analysis - name: cve-manager-analysis diff --git a/cve-vulner-manager/cve-py/config_yaml/secrets.yaml b/cve-vulner-manager/cve-py/config_yaml/secrets.yaml deleted file mode 100644 index eb2c021..0000000 --- a/cve-vulner-manager/cve-py/config_yaml/secrets.yaml +++ /dev/null @@ -1,20 +0,0 @@ ---- -apiVersion: secrets-manager.tuenti.io/v1alpha1 -kind: SecretDefinition -metadata: - name: cve-secrets-py -spec: - name: cve-secrets-py - keysMap: - db-pwdpy: - path: secrets/data/openeuler/cve-manager - key: db-pwdpy - db-uri: - path: secrets/data/openeuler/cve-manager - key: db-uri - cve-email-sendaddr: - path: secrets/data/openeuler/cve-manager - key: cve-email-sendaddr - cve-email-password: - path: secrets/data/openeuler/cve-manager - key: cve-email-password diff --git a/cve-vulner-manager/cve-py/config_yaml/service.yaml b/cve-vulner-manager/cve-py/config_yaml/service.yaml deleted file mode 100644 index 5d38198..0000000 --- a/cve-vulner-manager/cve-py/config_yaml/service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: cve-manager-analysis-service - namespace: cve-manager-analysis -spec: - ports: - - name: cve-manager-analysis - port: 8080 - protocol: TCP - targetPort: 8080 - selector: - app: cve-manager-analysis - type: ClusterIP -- Gitee