diff --git a/conf/app.conf b/conf/app.conf index 7482d247fe8f470e421948e61e5973cee365615c..2b23a07bd89f340aa819201ba0c4182abda90a02 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -60,7 +60,7 @@ genexcelflag = 2 genexcel = 0 */10 * * * * days = -30 prcnum = 50 -printlogflag = 2 +printlogflag = 1 printlog = 0 */20 * * * * @@ -104,6 +104,8 @@ openeulernum = 3000 cve_number = 2018 # Create an issue's warehouse whitelist;1: open; 2: close issue_whitelist = 2 +# List of affected branches +affected_branchs = openEuler-20.03.LTS [reflink] comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual.md diff --git a/conf/product_app.conf b/conf/product_app.conf index 9d51404aa5c115e8ae8fd3921e04ba74384e3af6..f6f7733f40030f180321eafcedf65e11e61daed7 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -99,6 +99,8 @@ openeulernum = 3000 cve_number = 2018 # Create an issue's warehouse whitelist;1: open; 2: close issue_whitelist = 2 +# List of affected branches +affected_branchs = openEuler-20.03-LTS [reflink] comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual.md diff --git a/controllers/hook.go b/controllers/hook.go index a5c0bbe3e61df14d750165632ab410fcd467bf05..29d629ec575ed3d676f5701fd52548ebefedc6cd 100644 --- a/controllers/hook.go +++ b/controllers/hook.go @@ -10,8 +10,10 @@ import ( "fmt" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" + "io/ioutil" "net/http" "os" + "regexp" "strconv" "strings" ) @@ -166,6 +168,7 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { if err != nil { return err } + issueTmp.StatusName = issueHook.Issue.StateName switch issueHook.State { case IssueOpenState: issueTmp.Status = 1 @@ -189,15 +192,28 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { } issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, unFix) case IssueCloseState: - issueTmp.Status = 3 - if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { - issueTmp.IssueStatus = 2 - cveCenter.IsExport = 3 - issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(unFix, fixed) + issueTmp.Status = 1 + cveCenter.IsExport = 0 + _, _, ok := checkIssueAnalysisComplete(&issueTmp) + if ok { + issueTmp.IssueStatus = 3 } else { - issueTmp.IssueStatus = 6 - cveCenter.IsExport = 2 - issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, unFix) + issueTmp.IssueStatus = 1 + } + issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, unFix) + issueTmp.StatusName = "open" + issuePrFlag := VerifyIssueAsPr(issueTmp, cveCenter) + if issuePrFlag { + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(unFix, fixed) + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, unFix) + } } case IssueRejectState: issueTmp.Status = 4 @@ -205,7 +221,6 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { cveCenter.IsExport = 2 issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, unFix) } - issueTmp.StatusName = issueHook.Issue.StateName appearErr := 0 err = models.UpdateIssueTemplate(&issueTmp, "status", "issue_status", "status_name", "issue_label") if err != nil { @@ -248,6 +263,300 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { return nil } +// When the issue status is complete, verify whether the pr is associated +func VerifyIssueAsPr(issueTmp models.IssueTemplate, cveCenter models.VulnCenter) bool { + sn := models.SecurityNotice{CveId: issueTmp.CveId} + secErr := sn.Read("cve_id") + if secErr != nil { + logs.Error("no data has been found, issueTmp: ", issueTmp) + return true + } + affectBranchsxList := []string{} + affectedBranchs := beego.AppConfig.String("cve::affected_branchs") + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } + token := beego.AppConfig.String("gitee::git_token") + //token := "8457c66db66955376519059b97e33dd1" + owner := beego.AppConfig.String("gitee::owner") + if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { + affectProductList := strings.Split(sn.AffectProduct, "/") + var branchMaps = make(map[string]bool) + for _, brands := range affectProductList { + if len(affectBranchsxList) > 0 { + for _, affectBranch := range affectBranchsxList { + if affectBranch == brands { + branchMaps[brands] = false + prList := getRepoBrandsAllPR(token, owner, brands, issueTmp.Repo) + if len(prList) > 0 { + for _, p := range prList { + issueFlagx := getPRRelatedBrandsAllIssue(token, owner, issueTmp.Repo, p.Number, issueTmp.IssueNum) + if issueFlagx { + branchMaps[brands] = issueFlagx + break + } + } + } + } + } + } + } + brandStr := "" + for brand, bv := range branchMaps { + if !bv { + logs.Error("brand: ", brand, ", pr is not related to issue, issueTmp: ", issueTmp) + brandStr = brandStr + brand + "/" + } + } + if brandStr != "" && len(brandStr) > 1 { + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, issueTmp) + if issueErr == nil { + commentBody := "Hey @" + issueTmp.Assignee + "\n" + + "关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #" + issueTmp.IssueNum + "\n" + + "分支: " + brandStr[:len(brandStr)-1] + "\n" + + "参考: " + "https://gitee.com/help/articles/4142" + "\n" + commentErr := createIssueComment(token, owner, issueTmp.Repo, cveCenter, issueTmp.IssueNum, commentBody) + if commentErr != nil { + logs.Error("创建issue评论失败, err: ", commentErr, ", tmp: ", issueTmp) + } + } + return false + } + } else { + unaffectedBranchList := []string{} + if issueTmp.AffectedVersion != "" && len(issueTmp.AffectedVersion) > 1 { + brandsGroup := strings.Split(issueTmp.AffectedVersion, ",") + if len(brandsGroup) > 0 { + for _, brand := range brandsGroup { + if brand == "" || len(brand) < 2 { + continue + } + brandList := strings.Split(brand, ":") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams != "受影响" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } else { + brandList = strings.Split(brand, ":") + if len(brandList) > 1 { + prams := strings.Replace(brandList[1], " ", "", -1) + if prams != "受影响" { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } + } + if len(brandList) == 1 { + unaffectedBranchList = append(unaffectedBranchList, brandList[0]) + } + } + } + } + branchStrs := "" + if len(unaffectedBranchList) > 0 { + for _, brands := range unaffectedBranchList { + if len(affectBranchsxList) > 0 { + for _, affectBranch := range affectBranchsxList { + if affectBranch == brands { + branchStrs = branchStrs + brands + "/" + } + } + } + } + } + if branchStrs != "" && len(branchStrs) > 1 { + branchStrs = branchStrs[:len(branchStrs)-1] + list, err := models.GetSecurityReviewerList() + if err != nil { + logs.Error(err) + return true + } + if len(list) == 0 { + logs.Error("list is null, issueTemp: ", issueTmp) + return true + } + anName := []string{} + for _, v := range list { + if v.Status == 1 { + anName = append(anName, "@" + v.NameSpace + " ") + } + } + if len(anName) > 0 { + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, issueTmp) + if issueErr == nil { + assignee := "Hey " + strings.Join(anName, ",") + commentBody := assignee + "\n" + + "关闭issue前,请确认分支: " + branchStrs + ": 受影响/不受影响, 如受影响,请联系maintainer: " + + issueTmp.Assignee + ",进行处理后,再关闭issue!" + commentErr := createIssueComment(token, owner, issueTmp.Repo, cveCenter, issueTmp.IssueNum, commentBody) + if commentErr != nil { + logs.Error("创建issue评论失败, err: ", commentErr, ", tmp: ", issueTmp) + } + } + } + } + } + return true +} + +// Create issue comment +func createIssueComment(accessToken, owner, path string, cve models.VulnCenter, issueNum, commentBody string) error { + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/issues/" + issueNum + "/comments" + + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "body": "%s" + }`, accessToken, commentBody) + logs.Info("create issue comment body: ", requestBody) + resp, err := util.HTTPPost(url, requestBody) + if err != nil { + logs.Error("创建issue评论失败, url: ", url, "cveId", cve.CveId, ",issueNum: ", issueNum, ",err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("创建issue评论失败, err: ", ok, "url: ", url) + return errors.New("创建issue评论失败") + } + } + return nil +} + +func getPRRelatedBrandsAllIssue(token, owner, repo string, num int, issueNum string) bool { + issueFlag := false + url := fmt.Sprintf(`https://gitee.com/api/v5/repos/%s/%s/pulls/%v/issues`, owner, repo, num) + pageSize := 20 + pageCount := 1 + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error(err) + return false + } + q := req.URL.Query() + q.Add("access_token", token) + q.Add("per_page", strconv.Itoa(pageSize)) + for { + q.Del("page") + q.Add("page", strconv.Itoa(pageCount)) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error(err) + break + } + if resp.StatusCode == http.StatusOK { + var il []models.HookIssue + read, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + logs.Error(err) + break + } + err = json.Unmarshal(read, &il) + if err != nil { + logs.Error(err) + break + } + for _, v := range il { + d, ok := isLegallyIssue(v) + if ok { + if issueNum == d.Number { + issueFlag = true + break + } + } + } + if len(il) < pageSize { + break + } + pageCount++ + } else { + resp.Body.Close() + break + } + } + return issueFlag +} + +// Verify that the current issue meets the requirements +func isLegallyIssue(i models.HookIssue) (pri models.PullRequestIssue, ok bool) { + if i.IssueType != "CVE和安全问题" || i.State != "closed" { + return + } + tt := strings.Trim(i.Title, " ") + regCveNum := regexp.MustCompile(`(?mi)CVE-[\d]{1,}-([\d]{1,})$`) + sm := util.RegexpCveNumber.FindAllStringSubmatch(i.Body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + val := sm[0][1] + tt = util.GetCveNumber(util.TrimString(val)) + if tt != "" && regCveNum.Match([]byte(tt)) { + ok = true + } + } + if ok { + pri.Id = i.Id + pri.Number = i.Number + pri.CveNumber = tt + pri.Repo = i.Repository.Path + } + return +} + +// Get the pr associated with a single warehouse +func getRepoBrandsAllPR(token, owner, brands, repo string) (prList []models.PullRequest) { + pageSize := 20 + pageCount := 1 + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/pulls", owner, repo) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error(err) + return + } + q := req.URL.Query() + q.Add("access_token", token) + q.Add("sort", "created") + q.Add("state", "merged") + q.Add("per_page", strconv.Itoa(pageSize)) + q.Add("base", brands) //target branch is openEuler-20.03-LTS + for { + q.Del("page") + q.Add("page", strconv.Itoa(pageCount)) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error(err) + break + } + if resp.StatusCode == http.StatusOK { + pr := make([]models.PullRequest, 0) + read, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error(err) + break + } + resp.Body.Close() + err = json.Unmarshal(read, &pr) + if err != nil { + logs.Error(err) + break + } + for _, v := range pr { + prList = append(prList, v) + } + if len(pr) < pageSize { + break + } + pageCount++ + } else { + resp.Body.Close() + break + } + } + return +} + func isNormalCloseIssue(cveID int64, issueState int8) bool { if issueState == 1 { return false diff --git a/models/modeldb.go b/models/modeldb.go index e7cd474b9b3833f5b94ead7d31f212a4fe46e523..691b0dfcc77cda6fbeae5d7736c4a132870a77e4 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -324,6 +324,7 @@ type OriginUpstreamConfigNode struct { type SecurityReviewer struct { Id int64 `orm:"pk;auto"` NameSpace string `orm:"unique" description:"码云空间地址"` + Status int8 `orm:"default(0);column(status)" description:"0: 全部;1:审核人"` } type IssueAssignee struct { diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index 4cb80f6af70af831bbd4768010c6e54a67a32014..9808d3973d902abf9ce474629722cc9f3e49cd9e 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -169,8 +169,6 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st var issueTemps models.IssueTemplate issueTemps.TemplateId = issTempID CreateIssueData(&issueTemps, cve, sc, resp, path, assignee, issueType, labels, owner) - // Store issue data - issTempIDx, idxErr := models.UpdateIssueTemplateAll(&issueTemps) if len(brandArray) > 0 { var brandArrayTmp []string for _, brand := range brandArray { @@ -179,9 +177,11 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st brandStr := strings.Join(brandArrayTmp, ",") issueTemp.AffectedVersion = brandStr } + // Store issue data + issTempIDx, idxErr := models.UpdateIssueTemplateAll(&issueTemps) if idxErr != nil { logs.Error("创建issue 模板的数据失败, cveNum: ", cve, ",err: ", err) - models.DeleteIssueTemplate(issTempID) + //models.DeleteIssueTemplate(issTempID) return "", err } logs.Info("创建issue 模板的数据成功, issTempID: ", issTempIDx, "cveNum: ", cve.CveNum) @@ -491,7 +491,7 @@ func CreateDepositHooks(accessToken string, owner string, path string, return nil } -func CreateIssueComment(accessToken, owner, path, Assignee string, +func CreateIssueComment(accessToken, owner, path, assignee string, cve models.VulnCenter, issResp map[string]interface{}, affectedVersion string) error { issueNum := issResp["number"].(string) if accessToken != "" && owner != "" && path != "" { @@ -502,7 +502,7 @@ func CreateIssueComment(accessToken, owner, path, Assignee string, return err } commentCmd := BConfig.String("reflink::comment_cmd") - commentBody := CommentTemplate(Assignee, commentCmd, affectedVersion) + commentBody := CommentTemplate(assignee, commentCmd, affectedVersion) requestBody := fmt.Sprintf(`{ "access_token": "%s", "body": "%s"