diff --git a/common/global.go b/common/global.go index d9b53b0e252fe5d13e268f1e1cf4b4d41a39fcc8..47f424d13ad5c71b1bb4d96cc40f8270919a6c4c 100644 --- a/common/global.go +++ b/common/global.go @@ -1,7 +1,6 @@ package common import ( - "fmt" "os" ) @@ -27,5 +26,5 @@ func InitGlobal() { gVal.GitPassword = os.Getenv("GITEE_PASSWORD") gVal.HookPwd = os.Getenv("HOOK_PWD") gVal.GitToken = os.Getenv("GITEE_TOKEN") - fmt.Println("env globalval: ", gVal) + //fmt.Println("env globalval: ", gVal) } \ No newline at end of file diff --git a/conf/app.conf b/conf/app.conf index ddf4f3252b2397712dba2aab105ef18bb2d906f3..c11b18fdae354ad12f74fae61d4e32b4c378af5c 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -61,7 +61,7 @@ genexcelflag = 2 genexcel = 0 */10 * * * * days = -30 prcnum = 50 -printlogflag = 1 +printlogflag = 2 printlog = 0 */20 * * * * @@ -118,4 +118,6 @@ forcerewrite = false snprefix = op-2020-10- snsuffix = 1002 # Version package excel download address -v_pack_excel_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv \ No newline at end of file +v_pack_20_03_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv +# Time difference in different time zones +sa_timestamp_zone = 28800 \ No newline at end of file diff --git a/conf/product_app.conf b/conf/product_app.conf index ee404c7c1d36e33395e4e7288452e689b811cd99..58da96caf1874cf6f74ec07db726a9443b74f831 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -115,4 +115,6 @@ forcerewrite = false snprefix = op-2020-10- snsuffix = 1002 # Version package excel download address -v_pack_excel_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv \ No newline at end of file +v_pack_20_03_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv +# Time difference in different time zones +sa_timestamp_zone = 28800 \ No newline at end of file diff --git a/controllers/hook.go b/controllers/hook.go index 112ad1814bd04943fa25f95ce28196bc3bd16e02..c199502a30e9a0fd8e8bb7eef018ec59bf51e74a 100644 --- a/controllers/hook.go +++ b/controllers/hook.go @@ -442,14 +442,9 @@ func VerifyIssueAsPr(issueTmp *models.IssueTemplate, cveCenter models.VulnCenter for _, affectBranch := range affectBranchsxList { if affectBranch == brands { branchMaps[brands] = false - prList := getRepoBrandsAllPR(token, owner, brands, issueTmp.Repo) + prList := getRepoIssueAllPR(affectBranch, token, owner, issueTmp.Repo, *issueTmp) if len(prList) > 0 { - for _, p := range prList { - issueFlagx := getPRRelatedBrandsAllIssue(token, owner, issueTmp.Repo, p.Number, issueTmp.IssueNum) - if issueFlagx { - branchMaps[brands] = issueFlagx - } - } + branchMaps[brands] = true } } } @@ -680,54 +675,59 @@ func isLegallyIssue(i models.HookIssue) (pri models.PullRequestIssue, ok bool) { } // Get the pr associated with a single warehouse -func getRepoBrandsAllPR(token, owner, brands, repo string) (prList []models.PullRequest) { - pageSize := 20 - pageCount := 1 - url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/pulls", owner, repo) +func getRepoIssueAllPR(affectBranch, token, owner, repo string, isTemp models.IssueTemplate) (prList []models.PullRequestIssue) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests", owner, isTemp.IssueNum) req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { - logs.Error(err) + logs.Error("NewRequest, url: ", url, ",err: ", err) return } q := req.URL.Query() q.Add("access_token", token) - q.Add("sort", "created") - q.Add("state", "merged") - q.Add("per_page", strconv.Itoa(pageSize)) - q.Add("base", brands) //target branch is openEuler-20.03-LTS - for { - q.Del("page") - q.Add("page", strconv.Itoa(pageCount)) - req.URL.RawQuery = q.Encode() - resp, err := http.DefaultClient.Do(req) + q.Add("repo", repo) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error("DefaultClient, url: ", url, ",err: ", err) + return + } + if resp.StatusCode == http.StatusOK { + issuePr := make([]map[string]interface{}, 0) + read, err := ioutil.ReadAll(resp.Body) if err != nil { - logs.Error(err) - break + logs.Error("ReadAll, url: ", url, ",err: ", err) + return } - if resp.StatusCode == http.StatusOK { - pr := make([]models.PullRequest, 0) - read, err := ioutil.ReadAll(resp.Body) - if err != nil { - logs.Error(err) - break - } - resp.Body.Close() - err = json.Unmarshal(read, &pr) - if err != nil { - logs.Error(err) - break - } - for _, v := range pr { - prList = append(prList, v) + resp.Body.Close() + err = json.Unmarshal(read, &issuePr) + if err != nil { + logs.Error("Unmarshal, url: ", url, ",err: ", err) + return + } + for _, v := range issuePr { + if _, ok := v["id"]; !ok { + continue } - if len(pr) < pageSize { - break + pr := models.PullRequestIssue{} + if v["state"].(string) == "merged" && v["mergeable"].(bool) { + if v["head"].(map[string]interface{})["label"].(string) == affectBranch || + v["base"].(map[string]interface{})["label"].(string) == affectBranch { + if v["head"].(map[string]interface{})["repo"].(map[string]interface{})["path"] == repo || + v["base"].(map[string]interface{})["repo"].(map[string]interface{})["path"] == repo { + if v["head"].(map[string]interface{})["repo"].(map[string]interface{})["namespace"].(map[string]interface{})["path"] == owner || + v["base"].(map[string]interface{})["repo"].(map[string]interface{})["namespace"].(map[string]interface{})["path"] == owner { + pr.Id = int64(v["id"].(float64)) + pr.Number = isTemp.IssueNum + pr.CveNumber = isTemp.CveNum + pr.Repo = repo + prList = append(prList, pr) + } + } + } } - pageCount++ - } else { - resp.Body.Close() - break } + } else { + resp.Body.Close() } return } diff --git a/cve-py/newexcels/20.03-CVE-2020-11-04.xlsx b/cve-py/newexcels/20.03-CVE-2020-11-04.xlsx deleted file mode 100644 index c1f456cd4292d535ad597f1e6b1f5bef2126d83c..0000000000000000000000000000000000000000 Binary files a/cve-py/newexcels/20.03-CVE-2020-11-04.xlsx and /dev/null differ diff --git a/cve-py/newexcels/cve-issue_2020-11-25.xlsx b/cve-py/newexcels/cve-issue_2020-11-25.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..3f58d6fb3d33db2c2976a9aa748aeef552b0a150 Binary files /dev/null and b/cve-py/newexcels/cve-issue_2020-11-25.xlsx differ diff --git a/models/cve.go b/models/cve.go index 7de90add36f891e8a5e34743348a07da993c692d..67c2d696edea7581c17579f493e966f19cfefbad 100644 --- a/models/cve.go +++ b/models/cve.go @@ -753,3 +753,12 @@ ORDER BY c.openeuler_score DESC _, err = o.Raw(sql).QueryRows(&list) return } + +func GetIssueNumber(packName string) (issueTemp []IssueTemplate, err error) { + sql := `select * from cve_issue_template where status = 3 and issue_status = 2 and +cve_id in (select cve_id from cve_vuln_center where cve_status = 2 and is_export = 3 and pack_name in ('%s'))` + sql = fmt.Sprintf(sql, packName) + o := orm.NewOrm() + _, err = o.Raw(sql).QueryRows(&issueTemp) + return +} \ No newline at end of file diff --git a/models/giteeissue.go b/models/giteeissue.go index b87a4f6a9564093c53343fdf68b9f3be922ee1d9..bc5b0eea2668357484638e39c793d659560a8c86 100644 --- a/models/giteeissue.go +++ b/models/giteeissue.go @@ -94,7 +94,8 @@ func (g *GiteOriginIssue) ParseToLoophole() (hole Loophole, err error) { lp.RepoDesc = g.RepoDesc scoreType, err := judgeScoreType(g.Body) if err != nil { - logs.Error(err) + logs.Error(err, "judgeScoreType, body: ", g.Body) + return lp, err } lp.ScoreType = scoreType if isNewTpl(g.Body) { @@ -145,6 +146,7 @@ func judgeScoreType(body string) (st string, err error) { if body == "" { return "", errors.New("can not judge score type by nil body") } + body = strings.ReplaceAll(body, " ", "") tb := []byte(body) vs := util.RegexpScoreTypeV2.Find(tb) if len(vs) > 0 { diff --git a/models/issue.go b/models/issue.go index 0010b6d3556dae7a94cf7343a3e6d7eb3bd78498..2d757b4991b77ae4ff403432e3d04f6bc7db31c4 100644 --- a/models/issue.go +++ b/models/issue.go @@ -157,14 +157,15 @@ func QueryIssueScoreRecord(cveId int64, status int8) (ScoreRecord, error) { func GetIssueTemplet(it *IssueTemplate) (localIt IssueTemplate, value bool) { o := orm.NewOrm() + cveId := it.CveId err := o.Raw("select *"+ - " from cve_issue_template where cve_id = ? ", it.CveId).QueryRow(&localIt) + " from cve_issue_template where cve_id = ? ", cveId).QueryRow(it) if err == nil { - logs.Info("cve_issue_template 查询结果：", localIt) - return localIt, true + logs.Info("cve_issue_template 查询结果：", it) + return *it, true } else { logs.Info("查询 cve_issue_template err, cveId: ", it.CveId, "err: ", err) - return localIt, false + return *it, false } } @@ -301,10 +302,7 @@ func DeleteIssueTemplate(issTempId int64) error { func CreateIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { o := orm.NewOrm() - var localIt IssueTemplate - errx := o.Raw("select *"+ - " from cve_issue_template where cve_num = ? and issue_num = ?", it.CveNum, it.IssueNum).QueryRow(&localIt) - if errx != nil || localIt.TemplateId == 0 { + if it.TemplateId == 0 { var issTempId int64 if issTempId, err = o.Insert(it); err == nil { logs.Info("insert cve_issue_template success, issTempId: ", issTempId, "cveNum: ", it.CveNum) @@ -314,7 +312,7 @@ func CreateIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { } return issTempId, nil } else { - it.TemplateId = localIt.TemplateId + it.TemplateId = it.TemplateId if num, err := o.Update(it); err == nil { logs.Info("update cve_issue_template success, num: ", num, "cveNum: ", it.CveNum) } else { diff --git a/task/issuetask.go b/task/issuetask.go index a374c404d8cd72a71df770fb39360df6e0e26987..58d3032ab819ba32243f13319e9517bd235c0c43 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -306,30 +306,30 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri // Query issue template var it models.IssueTemplate it.CveId = issueValue.CveId - lit, bools := models.GetIssueTemplet(&it) - if bools && lit.TemplateId > 0 { - lit.NVDScore = sr.NVDScore - lit.NVDVector = sr.NvectorVule - lit.CveBrief = issueValue.Description - lit.CveLevel = issueValue.CveLevel - if lit.Assignee == "" || len(lit.Assignee) < 2 { + _, bools := models.GetIssueTemplet(&it) + if bools && it.TemplateId > 0 { + it.NVDScore = sr.NVDScore + it.NVDVector = sr.NvectorVule + it.CveBrief = issueValue.Description + it.CveLevel = issueValue.CveLevel + if it.Assignee == "" || len(it.Assignee) < 2 { gitYaml, ok := models.QueryCveOpeneulerdata(issueValue.PackName, issueValue.CveVersion) if !ok || gitYaml.MainTainer == "" || len(gitYaml.MainTainer) < 1 { assignee, assErr := taskhandler.GetCollaboratorInfo(accessToken, owner, path) if assignee != "" && len(assignee) > 1 { - lit.Assignee = assignee + it.Assignee = assignee //return err } else { logs.Error("获取仓库： owner:", owner, "path:", path, "分析人失败", "assErr:", assErr, ", cveid: ", issueValue.CveId, ",创建无maintainer的issue") } } else { - lit.Assignee = gitYaml.MainTainer + it.Assignee = gitYaml.MainTainer } } - path = lit.Repo + path = it.Repo _, err := taskhandler.UpdateIssueToGit(accessToken, owner, path, - issueValue, lit) + issueValue, it) if err != nil { logs.Error("更新issue 模板失败， cveId: ", issueValue.CveId, "err: ", err) // Update issue status @@ -340,7 +340,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri models.UpdateIssueStatus(issueValue, 2) // Update score status models.UpdateIssueScore(issueValue, 2) - templetID, err := models.CreateIssueTemplate(&lit) + templetID, err := models.CreateIssueTemplate(&it) if err != nil { logs.Error("修改issue模板失败, cveId: ", issueValue.CveId, "err: ", err) return err diff --git a/taskhandler/common.go b/taskhandler/common.go index f754313f0a771cb4fb89f576ca78fd18a03d72ef..45809fe51d429db371f8cf912f14946efd3ca3e6 100644 --- a/taskhandler/common.go +++ b/taskhandler/common.go @@ -579,6 +579,7 @@ func RemoveSubstring(s string, subList []string) string { } } } + newStr = strings.TrimSpace(newStr) return newStr } diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index 2ee60b57ec4195f4392c4da00a30ea9d77a9cf07..5ad92c9156a0508af03526ce2caca5cd33abbe6f 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -91,25 +91,25 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st defer common.Catchs() var it models.IssueTemplate it.CveId = cve.CveId - its, err := models.GetIssueTemplet(&it) - if err && its.IssueNum != "" && len(its.IssueNum) > 0 { - if its.Assignee == "" || len(its.Assignee) == 0 { - its.Assignee = assignee + _, err := models.GetIssueTemplet(&it) + if err && it.IssueNum != "" && len(it.IssueNum) > 0 { + if it.Assignee == "" || len(it.Assignee) == 0 { + it.Assignee = assignee } - issueType := its.IssueType + issueType := it.IssueType labels := "" - if its.IssueLabel != "" && len(its.IssueLabel) > 1 { - labels = its.IssueLabel + if it.IssueLabel != "" && len(it.IssueLabel) > 1 { + labels = it.IssueLabel } else { labels = beego.AppConfig.String("labelUnFix") } if accessToken != "" && owner != "" && path != "" { - url := "https://gitee.com/api/v5/repos/" + owner + "/issues/" + its.IssueNum + url := "https://gitee.com/api/v5/repos/" + owner + "/issues/" + it.IssueNum score := strconv.FormatFloat(sc.NVDScore, 'f', 1, 64) - OpenEulerScore := strconv.FormatFloat(its.OpenEulerScore, 'f', 1, 64) - requestBody := CreateIssueBody(accessToken, owner, path, its.Assignee, - cve, sc, OpenEulerScore, score, labels, its, 1, its.IssueType, "", brandArray) + OpenEulerScore := strconv.FormatFloat(it.OpenEulerScore, 'f', 1, 64) + requestBody := CreateIssueBody(accessToken, owner, path, it.Assignee, + cve, sc, OpenEulerScore, score, labels, it, 1, it.IssueType, "", brandArray) if requestBody != "" && len(requestBody) > 1 { logs.Info("isssue_body: ", requestBody) resp, err := util.HTTPPatch(url, requestBody) @@ -124,24 +124,24 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st logs.Info("issue 创建成功，cveNum: ", cve.CveNum, "issueNum: ", resp["number"].(string)) // Structure data //var issueTemp models.IssueTemplate - CreateIssueData(&its, cve, sc, resp, path, its.Assignee, issueType, labels, owner) + CreateIssueData(&it, cve, sc, resp, path, it.Assignee, issueType, labels, owner) if len(brandArray) > 0 { var brandArryTmp []string for _, brand := range brandArray { brandArryTmp = append(brandArryTmp, brand+":") } brandStr := strings.Join(brandArryTmp, ",") - its.AffectedVersion = brandStr + it.AffectedVersion = brandStr } // Store issue data - issTempID, err := models.CreateIssueTemplate(&its) + issTempID, err := models.CreateIssueTemplate(&it) if err != nil { logs.Error("创建issue 模板的数据失败, cveNum: ", cve.CveNum, "err: ", err) return "", err } logs.Info("创建issue 模板的数据成功, issTempID: ", issTempID, "cveNum: ", cve.CveNum) } else { - logs.Info("不需要更新issue模板及issue状态, its: ", its) + logs.Info("不需要更新issue模板及issue状态, its: ", it) } // Update issue status models.UpdateIssueStatus(cve, 2) @@ -162,7 +162,7 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st url := "https://gitee.com/api/v5/repos/" + owner + "/issues" score := strconv.FormatFloat(sc.NVDScore, 'f', 1, 64) requestBody := CreateIssueBody(accessToken, owner, path, assignee, - cve, sc, "", score, labels, its, 2, issueType, "", brandArray) + cve, sc, "", score, labels, it, 2, issueType, "", brandArray) logs.Info("isssue_body: ", requestBody) if requestBody != "" && len(requestBody) > 1 { resp, err := util.HTTPPost(url, requestBody) diff --git a/taskhandler/cve.go b/taskhandler/cve.go index dd1e6323a341d2843e6c54a34724e844652cbbd8..38ffb1c825736ca96379cc866186e76c06b956ed 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -880,7 +880,7 @@ func GetCveOriginData(prcnum, days, openeulernum int, cveRef string) (bool, erro go func(idx int, cveData models.OriginUpstream) { ok, err := GenCveVuler(cveData, cveRef, openeulernum) if !ok { - logs.Error("cveData: ", cveData, "处理失败, err: ", err) + logs.Error("GenCveVuler, cveData: ", cveData, "处理失败, err: ", err) } ch <- idx }(i, cveOrg) @@ -914,7 +914,7 @@ func GetCveOriginExcelData(prcnum, days, openeulerNum int, cveRef string) (bool, go func(idx int, cveData models.OriginExcel) { ok, err := SyncCveVuler(cveData, cveRef, openeulerNum) if !ok { - logs.Error("cveData: ", cveData, "处理失败, err: ", err) + logs.Error("SyncCveVuler, cveData: ", cveData, "处理失败, err: ", err) } ch <- i }(i, cveOrg) @@ -1303,8 +1303,8 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c } var issueTemp models.IssueTemplate issueTemp.CveId = vul.CveId - localt, okl := models.GetIssueTemplet(&issueTemp) - if okl && localt.TemplateId > 0 { + _, okl := models.GetIssueTemplet(&issueTemp) + if okl && issueTemp.TemplateId > 0 { issueTemp.CveNum = cveData.CveNumber issueTemp.OwnedComponent = lop.Components issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) @@ -1468,7 +1468,7 @@ func GenCveVulerByIssue(cveData models.GiteOriginIssue, cveRef string, openeuler hole, err := cveData.ParseToLoophole() if err != nil { logs.Error("数据解析错误,") - models.UpdateCveStatusExportByNum(common.GetCurTime(), cveData.CveNumber, 2, 2, cveData.RepoPath) + //models.UpdateCveStatusExportByNum(common.GetCurTime(), cveData.CveNumber, 2, 2, cveData.RepoPath) models.UpdateCveIssueStatusById(3, cveData.Id) return false, err } @@ -1573,7 +1573,7 @@ func GetCveIssueData(prcnum, days, openeulernum int, cveRef, owner string) (bool go func(idx int, cveData models.GiteOriginIssue) { ok, err := GenCveVulerByIssue(cveData, cveRef, openeulernum, owner) if !ok { - logs.Error("cveData: ", cveData, "处理失败, err: ", err) + logs.Error("GenCveVulerByIssue, cveData: ", cveData, "处理失败, err: ", err) models.UpdateCveIssueStatusById(3, cveOrg.Id) } ch <- idx @@ -1607,22 +1607,22 @@ func GetCveSecurityNotice(cveNumber string) bool { //resp, err := http.Get(fmt.Sprintf(GetCveDetailUrl, cveNumber)) resp, err := http.Get(req.URL.String()) if err != nil { - logs.Error(err) + logs.Error("Get, url: ", req.URL.String(), err) return false } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil || body == nil { - logs.Error(err) + logs.Error("ReadAll, url: ", req.URL.String(), err) return false } var detail models.RespCveDetail err = json.Unmarshal(body, &detail) if err != nil { - logs.Error(err) + logs.Error("Unmarshal, url: ", req.URL.String(), err) return false } - logs.Info("url: ", req.URL.String(), "获取openEuler官网数据： ", detail) + if detail.Result != nil && detail.Result.Id > 0 { return true } @@ -1637,7 +1637,6 @@ func FilterCveExported() { logs.Error(err) return } - logs.Info("data: ", data) for _, v := range data { go func(center models.VulnCenter) { ewg.Add(1) @@ -1693,8 +1692,8 @@ func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { } fileName = filepath.Join(dir, fileName) du := "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" - //du := beego.AppConfig.String("excel::v_pack_excel_url") - du = beego.AppConfig.DefaultString("rpUrl", du) + //du := beego.AppConfig.String("excel::v_pack_20_03_url") + du = beego.AppConfig.DefaultString("excel::v_pack_20_03_url", du) localPath := filepath.Join(dir, "release-package.CSV") err = downloadPackageFile(localPath, du) if err != nil { diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 882077425fe00e61f519a63f57da868bb23a1d4a..30f10c741fe28dd087d11b125a699095fcd5b797 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -473,10 +473,9 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran for _, ab := range affectBranchsxList { if ab == affectBranch { affectBool = true - v.AffectProduct = affectBranch v.Introduction = strings.ReplaceAll(v.Introduction, v.AffectProduct, affectBranch) v.Theme = strings.ReplaceAll(v.Theme, v.AffectProduct, affectBranch) - break + v.AffectProduct = affectBranch } } } else { @@ -496,7 +495,7 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran //1.根据cve_num 获取issue_tpl 如果所有的issue_status ==2 0r issue_status == 6 则可以导出数据 list, err := models.GetIssueTplByCveNum(v.CveNum) if err != nil { - logs.Error(err) + logs.Error("GetIssueTplByCveNum, err: ", err) continue } mergerList := make([]string, 0) @@ -515,15 +514,15 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran if canMerger && len(mergerList) > 0 { canExport, err := models.GetCanExportCveDataSameNum(strings.Join(mergerList, ",")) if err != nil { - logs.Error(err) + logs.Error("GetCanExportCveDataSameNum, err: ", err) } ep := canExport[0] ep.SecID = v.SecID if ep.AffectProduct != "" && len(ep.AffectProduct) > 1 { if ep.AffectProduct != affectBranch { - ep.AffectProduct = affectBranch ep.Introduction = strings.ReplaceAll(ep.Introduction, ep.AffectProduct, affectBranch) ep.Theme = strings.ReplaceAll(ep.Theme, ep.AffectProduct, affectBranch) + ep.AffectProduct = affectBranch } } if len(canExport) > 1 { @@ -538,10 +537,9 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran for _, ab := range affectBranchsxList { if ab == affectBranch { affectBool = true - ex.AffectProduct = affectBranch ex.Introduction = strings.ReplaceAll(ex.Introduction, ex.AffectProduct, affectBranch) ex.Theme = strings.ReplaceAll(ex.Theme, ex.AffectProduct, affectBranch) - break + ex.AffectProduct = affectBranch } } } else { @@ -570,7 +568,6 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran fillLock.Unlock() } } - } } return nil @@ -773,19 +770,32 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I token := beego.AppConfig.String("gitee::git_token") //token := "8457c66db66955376519059b97e33dd1" owner := beego.AppConfig.String("gitee::owner") + // Time difference in different time zones + saTimeStampZone, ok := beego.AppConfig.Int64("excel::sa_timestamp_zone") + if ok != nil { + saTimeStampZone = 3600 * 8 + } //owner := "src-openeuler" st := util.TimeStrToInt(startTime, "2006-01-02") chData := make([]IssueAndPkg, 0) for _, v := range pkgList { - rt := util.TimeStrToInt(v.PubTime, "20060102 15-04-05") - prList := getRepoAllPR(affectBranch, token, owner, v.Repo, st, rt) - //get pull request related issue - repoIssue := make(map[int64]models.PullRequestIssue, 0) - for _, p := range prList { - getPRRelatedAllIssue(token, owner, v.Repo, st, rt, p.Number, repoIssue) + rt := util.TimeStrToInt(v.PubTime, "20060102 15-04-05") + saTimeStampZone + // 查询当前需要处理的issue + issueTemp, err := models.GetIssueNumber(v.Repo) + if err != nil || issueTemp == nil { + continue } - if len(repoIssue) > 0 { - chData = append(chData, IssueAndPkg{IssueMap: repoIssue, IssuePkg: v.Packages, Repo: v.Repo}) + for _, isTemp := range issueTemp { + prList := getRepoIssueAllPR(affectBranch, token, owner, v.Repo, st, rt, isTemp) + //get pull request related issue + repoIssue := make(map[int64]models.PullRequestIssue, 0) + for _, p := range prList { + //getPRRelatedAllIssue(token, owner, v.Repo, st, rt, p.Number, repoIssue) + repoIssue[p.Id] = p + } + if len(repoIssue) > 0 { + chData = append(chData, IssueAndPkg{IssueMap: repoIssue, IssuePkg: v.Packages, Repo: v.Repo}) + } } } c <- chData @@ -795,80 +805,82 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string) defer wgTrigger.Done() data := <-c var pkgList []string - for _, v := range data { - logs.Info("The SA currently being generated is: ", v) //parse package string to list pkgList = strings.Split(v.IssuePkg, " ") if len(pkgList) == 0 { + logs.Error("Data is filtered, v.IssuePkg: ", v.IssuePkg) continue } for _, iv := range v.IssueMap { tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo} err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo") if err != nil { - logs.Error("----", err) + logs.Error("GetIssueTemplateByColName, ----", err) continue } err = models.ReplacePackageByCveId(pkgList, tpl.CveId) if err != nil { - logs.Info(err) + logs.Info("ReplacePackageByCveId, err: ", err) continue } //save data to excel el, err := models.GetCanExportExcelData(tpl.CveNum, tpl.IssueNum) if err != nil { - logs.Error(err) + logs.Error("GetCanExportExcelData, err: ", err) return } err = ec.handleWriteContentSync(el, affectBranch) if err != nil { - logs.Error(err) + logs.Error("handleWriteContentSync, err: ", err) } } } } -func getRepoAllPR(affectBranch, token, owner, repo string, startTime, releaseTime int64) (prList []models.PullRequest) { - pageSize := 20 - pageCount := 1 - url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/pulls", owner, repo) +func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, + releaseTime int64, isTemp models.IssueTemplate) (prList []models.PullRequestIssue) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests", owner, isTemp.IssueNum) req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { - logs.Error(err) + logs.Error("NewRequest, url: ", url, ",err: ", err) return } q := req.URL.Query() q.Add("access_token", token) - q.Add("sort", "created") - q.Add("state", "merged") - q.Add("per_page", strconv.Itoa(pageSize)) - q.Add("base", affectBranch) //target branch is openEuler-20.03-LTS - for { - q.Del("page") - q.Add("page", strconv.Itoa(pageCount)) - req.URL.RawQuery = q.Encode() - resp, err := http.DefaultClient.Do(req) + q.Add("repo", repo) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error("DefaultClient, url: ", url, ",err: ", err) + return + } + if resp.StatusCode == http.StatusOK { + issuePr := make([]map[string]interface{}, 0) + read, err := ioutil.ReadAll(resp.Body) if err != nil { - logs.Error(err) - break + logs.Error("ReadAll, url: ", url, ",err: ", err) + return } - if resp.StatusCode == http.StatusOK { - pr := make([]models.PullRequest, 0) - read, err := ioutil.ReadAll(resp.Body) - if err != nil { - logs.Error(err) - break - } - resp.Body.Close() - err = json.Unmarshal(read, &pr) - if err != nil { - logs.Error(err) - break + resp.Body.Close() + err = json.Unmarshal(read, &issuePr) + if err != nil { + logs.Error("Unmarshal, url: ", url, ",err: ", err) + return + } + for _, v := range issuePr { + if _, ok := v["id"]; !ok { + continue } - for _, v := range pr { - mt := v.MergedAt.Local().Unix() - ct := v.ClosedAt.Local().Unix() + pr := models.PullRequestIssue{} + if v["state"].(string) == "merged" && v["mergeable"].(bool) { + //mt := v["closed_at"].(string).(time.Time).Local().Unix() + closedAt := v["closed_at"].(string) + mt := util.TimeStrToInt(closedAt[:19], "2006-01-02T15:04:05") + mergedAt := v["merged_at"].(string) + ct := util.TimeStrToInt(mergedAt[:19], "2006-01-02T15:04:05") + //logs.Info("******, pr: ", v, ",mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct) + //ct := v["merged_at"].(string).(time.Time).Local().Unix() var pt int64 if mt > 0 && ct > 0 { if mt > ct { @@ -877,103 +889,27 @@ func getRepoAllPR(affectBranch, token, owner, repo string, startTime, releaseTim pt = mt } if pt >= startTime && pt <= releaseTime { - prList = append(prList, v) + if v["head"].(map[string]interface{})["label"].(string) == affectBranch || + v["base"].(map[string]interface{})["label"].(string) == affectBranch { + if v["head"].(map[string]interface{})["repo"].(map[string]interface{})["path"] == repo || + v["base"].(map[string]interface{})["repo"].(map[string]interface{})["path"] == repo { + if v["head"].(map[string]interface{})["repo"].(map[string]interface{})["namespace"].(map[string]interface{})["path"] == owner || + v["base"].(map[string]interface{})["repo"].(map[string]interface{})["namespace"].(map[string]interface{})["path"] == owner { + pr.Id = int64(v["id"].(float64)) + pr.Number = isTemp.IssueNum + pr.CveNumber = isTemp.CveNum + pr.Repo = repo + prList = append(prList, pr) + } + } + } } } } - if len(pr) < pageSize { - break - } - pageCount++ - } else { - resp.Body.Close() - break } + } else { + resp.Body.Close() } return } -func getPRRelatedAllIssue(token, owner, repo string, startTime, releaseTime int64, num int, issueList map[int64]models.PullRequestIssue) { - if issueList == nil { - return - } - url := fmt.Sprintf(`https://gitee.com/api/v5/repos/%s/%s/pulls/%v/issues`, owner, repo, num) - pageSize := 20 - pageCount := 1 - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - logs.Error(err) - return - } - q := req.URL.Query() - q.Add("access_token", token) - q.Add("per_page", strconv.Itoa(pageSize)) - for { - q.Del("page") - q.Add("page", strconv.Itoa(pageCount)) - req.URL.RawQuery = q.Encode() - resp, err := http.DefaultClient.Do(req) - if err != nil { - logs.Error(err) - break - } - if resp.StatusCode == http.StatusOK { - var il []models.HookIssue - read, err := ioutil.ReadAll(resp.Body) - resp.Body.Close() - if err != nil { - logs.Error(err) - break - } - err = json.Unmarshal(read, &il) - if err != nil { - logs.Error(err) - break - } - for _, v := range il { - d, ok := isLegallyIssue(v, startTime, releaseTime) - if ok { - issueList[d.Id] = d - } - } - if len(il) < pageSize { - break - } - pageCount++ - } else { - resp.Body.Close() - break - } - } -} - -func isLegallyIssue(i models.HookIssue, startTime int64, releaseTime int64) (pri models.PullRequestIssue, ok bool) { - if i.IssueType != IssueType || i.State != "closed" { - return - } - ft := i.FinishedAt.Unix() - if startTime > ft || ft > releaseTime { - return - } - tt := strings.Trim(i.Title, " ") - regCveNum := regexp.MustCompile(`(?mi)CVE-[\d]{1,}-([\d]{1,})$`) - /*if tt != "" && regCveNum.Match([]byte(tt)) { - ok = true - } else {*/ - sm := util.RegexpCveNumber.FindAllStringSubmatch(i.Body, -1) - if len(sm) > 0 && len(sm[0]) > 0 { - val := sm[0][1] - tt = util.GetCveNumber(util.TrimString(val)) - if tt != "" && regCveNum.Match([]byte(tt)) { - ok = true - } - } - //} - if ok { - pri.Id = i.Id - pri.Number = i.Number - pri.CveNumber = tt - pri.Repo = i.Repository.Path - } - return -} diff --git a/taskhandler/grabissue.go b/taskhandler/grabissue.go index 1497ae46a1aabdcb41c80d0ebab1584bc2031c61..300466813070434fc546329d2613cdcea80ff772 100644 --- a/taskhandler/grabissue.go +++ b/taskhandler/grabissue.go @@ -4,6 +4,7 @@ import ( "cvevulner/models" "cvevulner/util" "encoding/json" + "errors" "fmt" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" @@ -23,7 +24,7 @@ const ( //GiteRepoBranch get repo branch url GiteRepoBranch = `https://gitee.com/api/v5/repos/%v/%v/branches?access_token=%v` //RepoInfoURL get repo info url - RepoInfoURL = "https://api.openeuler.org/pkgmanagedebug/packages/packageInfo?table_name=mainline&pkg_name=%s" + RepoInfoURL = "https://api.openeuler.org/pkgmanage/packages/packageInfo?table_name=openEuler_LTS_20.03&pkg_name=%s" perPage = 50 //IssueType Types of issues crawled IssueType = "CVE和安全问题" @@ -69,16 +70,16 @@ type Info struct { //GrabIssueByOrg grab issue by org name func GrabIssueByOrg(accToken, org string) error { - logs.Info("grab issue start......") + logs.Info("Synchronize gitee's issue start......") orgInfo, err := GetOrgInfo(accToken, org) if err != nil { - logs.Error(err) + logs.Error("GetOrgInfo, org: ", org, ",err: ", err) return err } reposNum := orgInfo.PublicRepos + orgInfo.PrivateRepos if reposNum <= 0 { logs.Info(fmt.Sprintf("%v cantain %v repository,grab issue finish!", org, reposNum)) - return err + return errors.New(fmt.Sprintf("%v cantain %v repository,grab issue finish!", org, reposNum)) } pageSize := reposNum / int64(perPage) if reposNum%int64(perPage) > 0 { @@ -89,7 +90,7 @@ func GrabIssueByOrg(accToken, org string) error { go GetOrgRepos(accToken, org, i) } wg.Wait() - logs.Info("grab issue finish...") + logs.Info("Synchronize gitee's issue finish...") return nil } @@ -98,13 +99,13 @@ func GrabIssueByRepo(accToken, owner, repo, state string) { page := 1 product, err := getInfProduct(accToken, owner, repo) if err != nil { - logs.Error(err) + logs.Error("getInfProduct, err: ", err) } desc := GetRepoDescription(repo) for { list, err := GetIssueList(accToken, owner, repo, state, page) if err != nil { - logs.Error(err) + logs.Error("GetIssueList, repo: ", repo, ",err: ", err) break } handleIssueList(list, product, desc) @@ -118,16 +119,19 @@ func GrabIssueByRepo(accToken, owner, repo, state string) { func getInfProduct(token string, owner string, repo string) (infPro string, err error) { resp, err := http.Get(fmt.Sprintf(GiteRepoBranch, owner, repo, token)) if err != nil { + logs.Error("url: ", GiteRepoBranch, ",repo:", repo, ",err: ", err) return "", err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { + logs.Error("ReadAll: ", GiteRepoBranch, ",repo:", repo, ",err: ", err) return "", err } var branchList []Branch err = json.Unmarshal(body, &branchList) if err != nil { + logs.Error("Unmarshal: ", GiteRepoBranch, ",repo:", repo, ",err: ", err) return "", err } affectBranchsxList := []string{} @@ -210,19 +214,19 @@ func GetOrgRepos(accToken, org string, page int64) { defer wg.Done() resp, err := http.Get(fmt.Sprintf(GiteOrgReposURL, org, accToken, page, perPage)) if err != nil { - logs.Error(err) + logs.Error("Get, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) return } defer resp.Body.Close() var reps []models.HookRepository body, err := ioutil.ReadAll(resp.Body) if err != nil { - logs.Error(err) + logs.Error("ReadAll, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) return } err = json.Unmarshal(body, &reps) if err != nil { - logs.Error(err) + logs.Error("Unmarshal, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) return } for _, v := range reps { @@ -234,16 +238,21 @@ func GetOrgRepos(accToken, org string, page int64) { func GetIssueList(accToken, owner, repo, state string, page int) (issueList []models.HookIssue, err error) { resp, err := http.Get(fmt.Sprintf(GiteRepoIssuesURL, owner, repo,accToken, state, page, perPage)) if err != nil { + logs.Error("Get, GiteRepoIssuesURL: ", GiteRepoIssuesURL, ", repo: ", repo, ", err: ", err) return issueList, err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { + logs.Error("ReadAll, GiteRepoIssuesURL: ", GiteRepoIssuesURL, ", repo: ", repo, ", err: ", err) return issueList, err } // //logs.Error(string(body)) err = json.Unmarshal(body, &issueList) + if err != nil { + logs.Error("Unmarshal, GiteRepoIssuesURL: ", GiteRepoIssuesURL, ", repo: ", repo, ", err: ", err) + } return } @@ -255,16 +264,19 @@ func GetRepoDescription(repo string) (desc string) { url := fmt.Sprintf(RepoInfoURL, repo) resp, err := http.Get(url) if err != nil { + logs.Error("Get, RepoInfoURL: ", RepoInfoURL, ",err: ", err) return "" } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { + logs.Error("ReadAll, RepoInfoURL: ", RepoInfoURL, ",err: ", err) return "" } var pkg PackageInfo err = json.Unmarshal(body, &pkg) if err != nil { + logs.Error("Unmarshal, RepoInfoURL: ", RepoInfoURL, ",err: ", err) return "" } if pkg.Code == "2001" { diff --git a/util/parsepayload.go b/util/parsepayload.go index 6aa9e595d90d3eb1fca6e712fdbe7f8152759a85..1a892c199ef0ce5f0a881104c58d1ceb15849022 100644 --- a/util/parsepayload.go +++ b/util/parsepayload.go @@ -50,9 +50,9 @@ var ( //RegexpCveComponents components extract regexp RegexpCveComponents = regexp.MustCompile(`漏洞归属组件[:：](?s:(.*?))漏洞归属的?版本[:：]`) //RegexpCveVersion cveVersion extract regexp - RegexpCveVersion = regexp.MustCompile(`漏洞归属的?版本[:：](?s:(.*?))CVSS V[23].0分值[:：]`) + RegexpCveVersion = regexp.MustCompile(`漏洞归属的?版本[:：](?s:(.*?))CVSS [Vv][23].[0-9xX]分值[:：]`) //RegexpCveScore cveScore extract regexp - RegexpCveScore = regexp.MustCompile(`CVSS V[23].0分值[:：](?s:(.*?))漏洞[简描]述[:：]`) + RegexpCveScore = regexp.MustCompile(`CVSS [Vv][23].[0-9xX]分值[:：](?s:(.*?))漏洞[简描]述[:：]`) //RegexpCveBriefDesc brief description extract regexp RegexpCveBriefDesc = regexp.MustCompile(`漏洞[简描]述[:：](?s:(.*?))影响性分析说明[:：]`) //RegexpCveBriefDesc new tpl brief description extract regexp @@ -83,8 +83,8 @@ var ( //RegexpSpecialDigital = regexp.MustCompile(`(cvssv[1-9].[0-9]|CVSSV[1-9].[0-9]|CVSS[:：][1-9].[0-9]|cvss[:：][1-9].[0-9]|[1-9].[0-9]/|[1-9].[0-9] /)*`) //^((CVSS：3.0|CVSS：2.0|3.0/|2.0/|3.0 /|2.0 /).)*$ RegexpVector = regexp.MustCompile(`AV:[NLAP](?s:(.*?))/A:[LNH]`) RegexpVectorV2 = regexp.MustCompile(`AV:[LAN](?s:(.*))/Au:[MSN](?s:(.*))/A:[NPC]`) - RegexpScoreTypeV2 = regexp.MustCompile(`(?mi)^CVSS v2.0分值\s*`) - RegexpScoreTypeV3 = regexp.MustCompile(`(?mi)^CVSS v3.0分值\s*`) + RegexpScoreTypeV2 = regexp.MustCompile(`(?mi)^CVSS[vV]2.[0-9xX]\s*`) // CVSS V3.0分值: + RegexpScoreTypeV3 = regexp.MustCompile(`(?mi)^CVSS[vV]3.[0-9xX]\s*`) RegexpIsNewTpl = regexp.MustCompile(`(?mi)^原理分析[：:]\s*`) RegexpIsNewTpl2 = regexp.MustCompile(`(?mi)^规避方案或消减措施[:：]\s*`) regexpEffectVersion = regexp.MustCompile(`(?mi)[\d]{1,}\.(.*?)[:：]受影响`)