From 550d771dcd461059903abc1e94a5a852e8776dac Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Sat, 30 Jan 2021 17:02:02 +0800 Subject: [PATCH] Manual creation of issues, optimization of solutions with low data synchronization efficiency --- conf/app.conf | 4 +- conf/product_app.conf | 2 + controllers/hook.go | 137 ++++++++++++++++++++++ models/giteeissue.go | 40 ++++++- models/hookevent.go | 12 +- models/uploadcve.go | 14 +++ task/hook.go | 25 +++- task/inittask.go | 16 ++- task/issuetask.go | 8 +- taskhandler/createissue.go | 232 ++++++++++++++++++------------------- taskhandler/grabissue.go | 10 +- taskhandler/hook.go | 111 ++++++++++++++++++ 12 files changed, 467 insertions(+), 144 deletions(-) diff --git a/conf/app.conf b/conf/app.conf index 32a0af8..016e2d3 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -47,7 +47,7 @@ maxsize=204800 [crontab] ymalflag = 2 getymal = 0 0 1 * * 1 -cveflag = 1 +cveflag = 2 getcve = 0 4 16 * * * oricveflag = 2 oricvecheck = 0 16 19 * * * @@ -68,6 +68,8 @@ unlockflag = 2 unlock = 0 30 19 * * * hookflag = 2 prochook = 0 34 17 * * * +createhookflag = 2 +createhook = 0 29 11 * * * exceptflag = 2 exceptcve = 0 3 19 * * * gensaflag = 2 diff --git a/conf/product_app.conf b/conf/product_app.conf index 84d7f5d..a331063 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -69,6 +69,8 @@ unlockflag = 1 unlock = 0 0 12 * * * hookflag = 1 prochook = 0 1 8 * * * +createhookflag = 1 +createhook = 0 20 2 * * * exceptflag = 2 exceptcve = 0 0 4 * * * gensaflag = 1 diff --git a/controllers/hook.go b/controllers/hook.go index 7985d2b..ec5359a 100644 --- a/controllers/hook.go +++ b/controllers/hook.go @@ -4,6 +4,7 @@ import ( "bytes" "cvevulner/common" "cvevulner/models" + "cvevulner/task" "cvevulner/taskhandler" "cvevulner/util" "encoding/json" @@ -179,6 +180,18 @@ func (c *HookEventControllers) handleIssue() { logs.Error(err) } } + if issueHook.Action == "open" { + err = gitAddIssueProc(&issueHook) + if err != nil { + logs.Error(err) + } + } + if issueHook.Action == "delete" { + err = gitDelIssueProc(&issueHook) + if err != nil { + logs.Error(err) + } + } } func closeIssuePrivilage(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, @@ -2012,3 +2025,127 @@ func UpdateGiteIssue(url string, param []byte) bool { } return false } + +func AddGitIssue(issueHook *models.IssuePayload, desc, product string) { + issueTitle := util.TrimString(issueHook.Title) + issueType := util.TrimString(issueHook.Issue.TypeName) + issueNumber := util.TrimString(issueHook.Issue.Number) + issueState := util.TrimString(issueHook.Issue.State) + issueZhState := util.TrimString(issueHook.Issue.StateName) + repoPath := util.TrimString(issueHook.Repository.Path) + if issueType == CIssueType || strings.HasPrefix(issueTitle, "CVE") { + item := models.GiteOriginIssue{IssueId: issueHook.Issue.Id, Url: issueHook.Issue.HtmlUrl, + Number: issueNumber, State: issueState, Title: issueTitle, + IssueType: issueType, SecurityHole: true, + IssueCreateAt: issueHook.Issue.CreateAt, IssueUpdateAt: issueHook.Issue.UpdateAt, + IssueFinishAt: issueHook.Issue.FinishedAt, IssueCreate: issueHook.Issue.User.Login, + IssueAssignee: "", RepoPath: repoPath, + RepoUrl: issueHook.Repository.Url, InfProduct: product, + RepoDesc: desc, IssueState: issueZhState} + //vt := util.TrimString(v.Title) + if strings.HasPrefix(issueTitle, "CVE") { + item.CveNumber = issueTitle + } else if issueHook.Issue.Body != "" { + //通过正则表达式去body中截取 稍后完善 + sm := util.RegexpCveNumber.FindAllStringSubmatch(issueHook.Issue.Body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + item.CveNumber = util.TrimString(sm[0][1]) + } + } + //vb := util.TrimString(v.Body) + vb := strings.ReplaceAll(issueHook.Issue.Body, "：", ":") + item.Body = vb + issueTmp := models.IssueTemplate{} + issueTmp.CveNum = item.CveNumber + issueTmp.OwnedComponent = repoPath + issueErr := models.GetIssueTemplateByColName(&issueTmp, "CveNum", "OwnedComponent") + if issueErr == nil { + item.IssueExistTpl = true + } else { + item.IssueExistTpl = false + } + err := item.InsertOrUpdate(2) + if err != nil { + logs.Error("insert or update issue fail:", err) + } + } +} + +func DelOrgIssue(issueHook *models.IssuePayload) { + issueTitle := util.TrimString(issueHook.Title) + issueType := util.TrimString(issueHook.Issue.TypeName) + issueNumber := util.TrimString(issueHook.Issue.Number) + issueState := util.TrimString(issueHook.Issue.State) + issueZhState := util.TrimString(issueHook.Issue.StateName) + repoPath := util.TrimString(issueHook.Repository.Path) + if issueType == CIssueType || strings.HasPrefix(issueTitle, "CVE") { + item := models.GiteOriginIssue{IssueId: issueHook.Issue.Id, Url: issueHook.Issue.HtmlUrl, + Number: issueNumber, State: issueState, Title: issueTitle, + IssueType: issueType, SecurityHole: true, + IssueCreateAt: issueHook.Issue.CreateAt, IssueUpdateAt: issueHook.Issue.UpdateAt, + IssueFinishAt: issueHook.Issue.FinishedAt, IssueCreate: issueHook.Issue.User.Login, + IssueAssignee: "", RepoPath: repoPath, + RepoUrl: issueHook.Repository.Url, InfProduct: "", + RepoDesc: "", IssueState: issueZhState} + if strings.HasPrefix(issueTitle, "CVE") { + item.CveNumber = issueTitle + } else if issueHook.Issue.Body != "" { + //通过正则表达式去body中截取 稍后完善 + sm := util.RegexpCveNumber.FindAllStringSubmatch(issueHook.Issue.Body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + item.CveNumber = util.TrimString(sm[0][1]) + } + } + err := item.Detlete() + if err != nil { + logs.Error("Failed to delete issue:", err) + } + } +} + +// Entry function for handling issue status +func gitAddIssueProc(issueHook *models.IssuePayload) error { + token := beego.AppConfig.String("gitee::git_token") + owner := beego.AppConfig.String("gitee::owner") + path := issueHook.Issue.Repository.Path + // The amount of data processed at a time + prcNum, err := beego.AppConfig.Int("crontab::prcnum") + if err != nil { + logs.Error("crontab::prcnum, error: ", err) + return errors.New("value is nil") + } + // How many days have been processed to date data + days, ok := beego.AppConfig.Int("crontab::days") + if ok != nil { + logs.Error("crontab::days, err:", err) + return ok + } + // openeuler Number start value + cveRef := beego.AppConfig.String("cve::cveref") + openeulerNum, ok := beego.AppConfig.Int("cve::openeulernum") + if ok != nil { + logs.Error("cve::openeulernum, err:", err) + return ok + } + product, err := taskhandler.GetInfProduct(token, owner, path) + if err != nil { + logs.Error("GetInfProduct, err: ", err) + } + desc := taskhandler.GetRepoDescription(path) + AddGitIssue(issueHook, desc, product) + // Compatible with created issue data + oki, err := taskhandler.GetCveIssueData(prcNum, days, openeulerNum, cveRef, owner) + if !oki { + logs.Error("ProcCveOriginData, GetCveIssueData, err: ", err) + } + createErr := task.CreateIssue() + if createErr != nil { + logs.Error("Failed to synchronize issue data of cve, err: ", createErr) + } + return createErr +} + +func gitDelIssueProc(issueHook *models.IssuePayload) error { + DelOrgIssue(issueHook) + return nil +} diff --git a/models/giteeissue.go b/models/giteeissue.go index 9130ec6..69d49bc 100644 --- a/models/giteeissue.go +++ b/models/giteeissue.go @@ -63,18 +63,46 @@ func (g GiteOriginIssue) Update() error { return err } -func (g *GiteOriginIssue) InsertOrUpdate() error { +func (g *GiteOriginIssue) InsertOrUpdate(flag int) error { + if g.Number == "" { + return errors.New("issue number can not be null") + } + o := orm.NewOrm() + if flag == 1 { + ex := GiteOriginIssue{Number: g.Number, CveNumber: g.CveNumber, RepoPath: g.RepoPath, IssueId: g.IssueId} + err := o.Read(&ex, "number", "cve_number", "repo_path", "issue_id") + if err == nil && ex.Id > 0 { + g.Id = ex.Id + } + logs.Info("processing, repo: ", g.RepoPath, g.Number, g.CveNumber) + _, err = o.InsertOrUpdate(g) + return err + } else { + ex := GiteOriginIssue{CveNumber: g.CveNumber, RepoPath: g.RepoPath} + err := o.Read(&ex, "cve_number", "repo_path") + if err == nil && ex.Id > 0 { + g.Id = ex.Id + logs.Warn("repo: ", g.RepoPath, ",cveNumber: ", g.CveNumber, + ",cve has already been created an issue and cannot be created again") + return nil + } + logs.Info("processing, repo: ", g.RepoPath, g.Number, g.CveNumber) + _, err = o.InsertOrUpdate(g) + return err + } +} + +func (g *GiteOriginIssue) Detlete() error { if g.Number == "" { return errors.New("issue number can not be null") } o := orm.NewOrm() ex := GiteOriginIssue{Number: g.Number, CveNumber: g.CveNumber, RepoPath: g.RepoPath, IssueId: g.IssueId} - err := o.Read(&ex, "number", "cve_number", "repo_path", "issue_id") - if err == nil && ex.Id > 0 { - g.Id = ex.Id + num, err := o.Delete(&ex, "number", "cve_number", "repo_path", "issue_id") + if err == nil && num > 0 { + logs.Info("GiteOriginIssue, successfully deleted, "+ + "Number: ", g.Number, ",CveNumber:", g.CveNumber, ",RepoPath:", g.RepoPath, ",IssueId:", g.IssueId) } - logs.Info("processing, repo: ", g.RepoPath, g.Number, g.CveNumber) - _, err = o.InsertOrUpdate(g) return err } diff --git a/models/hookevent.go b/models/hookevent.go index 51d760b..83f5c65 100644 --- a/models/hookevent.go +++ b/models/hookevent.go @@ -23,10 +23,10 @@ type HookIssue struct { Number string Title string Labels []IssueLabel `json:"labels"` - State string - HtmlUrl string `json:"html_url"` //Comment on the url on the code cloud - Body string - StateName string `json:"state_name"` + State string `json:"state"` + HtmlUrl string `json:"html_url"` //Comment on the url on the code cloud + Body string `json:"body"` + StateName string `json:"state_name"` User HookUser Assignee HookUser Repository HookRepository @@ -34,6 +34,7 @@ type HookIssue struct { UpdateAt time.Time `json:"updated_at"` FinishedAt time.Time `json:"finished_at"` IssueType string `json:"issue_type"` + TypeName string `json:"type_name"` SecurityHole bool `json:"security_hole"` IssueState string `json:"issue_state"` } @@ -50,7 +51,7 @@ type HookRepository struct { Id int64 FullName string `json:"full_name"` Url string `json:"url"` - Path string + Path string `json:"path"` Name string Owner HookUser Private bool @@ -102,6 +103,7 @@ type IssuePayload struct { Description string //issue description State string //issue status Url string //issue URL on code cloud + Repository HookRepository } type PullRequest struct { diff --git a/models/uploadcve.go b/models/uploadcve.go index 801e41e..e486f36 100644 --- a/models/uploadcve.go +++ b/models/uploadcve.go @@ -48,6 +48,19 @@ func QueryOriginRepo(pkgName string) (GiteRepo, bool) { } } +func QueryOrgAllRepo(org string) ([]GiteRepo) { + o := orm.NewOrm() + var gr []GiteRepo + num, err := o.Raw("select * from cve_gite_repo where org_path = ? and status = ?", org, 0).QueryRows(&gr) + if num == 0 { + logs.Info("QueryOrgAllRepo, cve_gite_repo"+", repoOrg: ", org, ", No corresponding data, err: ", err) + return gr + } else { + logs.Info("QueryOrgAllRepo, cve_gite_repo"+", repoOrg: ", org, ", search successful") + return gr + } +} + func QueryRepoMember(groupId int64, memberType string) ([]GiteRepoMember, bool) { o := orm.NewOrm() var grm []GiteRepoMember @@ -931,3 +944,4 @@ func GetOpengaussYaml(opy *OpenGussYaml, colName ...string) error { err := o.Read(opy, colName...) return err } + diff --git a/task/hook.go b/task/hook.go index cacccfb..8b308a5 100644 --- a/task/hook.go +++ b/task/hook.go @@ -11,7 +11,7 @@ import ( func ProcHookEvent() error { defer common.Catchs() - logs.Info("处理hook事件 task start") + logs.Info("Remove duplicate webhook event task start") // Query the cve to be processed, 1: add; 2: modify BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -26,6 +26,27 @@ func ProcHookEvent() error { pwd := BConfig.String("hook::hookpwd") // Get the data source of the table err = taskhandler.ProcHookEvent(hookurl, owner, accessToken, pwd, gaussOwner, gitGaussToken) - logs.Info("处理hook事件 task end") + logs.Info("Remove duplicate webhook event task ends") + return err +} + +// Create webhooks for all repo +func CreateWebhookEvent() error { + defer common.Catchs() + logs.Info("Create webhook event start") + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return err + } + hookurl := BConfig.String("hook::hookurl") + owner := BConfig.String("gitee::owner") + accessToken := os.Getenv("GITEE_TOKEN") + gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") + gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") + pwd := BConfig.String("hook::hookpwd") + // Get the data source of the table + err = taskhandler.CreateHookEvent(hookurl, owner, accessToken, pwd, gaussOwner, gitGaussToken) + logs.Info("End of creating webhook event") return err } diff --git a/task/inittask.go b/task/inittask.go index f707b8d..2db2c86 100644 --- a/task/inittask.go +++ b/task/inittask.go @@ -70,6 +70,14 @@ func ProcHookTask(prochook string) { logs.Info("The execution of the Hook event task ends...") } +// Create webhooks for all repo +func CreateHookTask(createHook string) { + logs.Info("Start creating all webhook tasks...") + createhook := toolbox.NewTask("CreateWebhookEvent", createHook, CreateWebhookEvent) + toolbox.AddTask("CreateWebhookEvent", createhook) + logs.Info("End of execution to create all webhook tasks...") +} + //GenSAExcelTask Execute issue to generate excel task start func GenSAExcelTask(genExcel string) { logs.Info("Execute issue to generate excel task start...") @@ -205,8 +213,12 @@ func InitTask() bool { if hookflag == 1 && errx == nil { prochook := BConfig.String("crontab::prochook") ProcHookTask(prochook) - } else { - logs.Info("prochook no task") + } + // Create webhooks for all repo + createHookFlag, errx := BConfig.Int("crontab::createhookflag") + if createHookFlag == 1 && errx == nil { + createHook := BConfig.String("crontab::createhook") + CreateHookTask(createHook) } // Handling abnormal cve data exceptflag, errx := BConfig.Int("crontab::exceptflag") diff --git a/task/issuetask.go b/task/issuetask.go index 62795e1..21ab8bb 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -122,7 +122,7 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears // add mutex lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) if !lockErr { - logs.Error("The current cve is processing, continue to process the next data, " + + logs.Error("The current cve is processing, continue to process the next data, "+ "err: ", lockErr, ",data: ", issueValue) continue } @@ -501,7 +501,11 @@ func ProcIssue(issueValue models.VulnCenter, } } legalFlag := false - if issueValue.DataSource == 3 { + if issueValue.DataSource == 4 { + if len(issueValue.CveNum) < 2 || len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + legalFlag = true + } + } else if issueValue.DataSource == 3 { if len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { legalFlag = true diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index d3fdda7..5d0305a 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -250,7 +250,7 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, //models.DeleteIssueTemplate(issTempID) return "", errors.New("内部错误, 模板存储失败") } - logs.Info("The data of the issue template was successfully " + + logs.Info("The data of the issue template was successfully "+ "created, issTempID: ", issTempIDx, "cveNum: ", cve.CveNum) // Create issue comment affectedVersion := "" @@ -392,9 +392,9 @@ func UpdateIssueToGit(accessToken string, owner string, path string, return "", nil } -func CreateIssueHookData(issHook *models.IssueHooks, cve models.VulnCenter, resp map[string]interface{}, - path, owner string, issueNum string, issueId int64) *models.IssueHooks { - issHook.CveId = cve.CveId +func CreateIssueHookData(issHook *models.IssueHooks, resp map[string]interface{}, + path, owner string, issueNum string, issueId, cveId int64) *models.IssueHooks { + issHook.CveId = cveId issHook.IssueId = issueId issHook.IssueNum = issueNum issHook.HookId = int64(resp["id"].(float64)) @@ -432,6 +432,97 @@ func CreateIssueHookData(issHook *models.IssueHooks, cve models.VulnCenter, resp return issHook } +func PatchWebHooks(accessToken, owner, path, hookurl, pwd, issueNum, hookId string, issueId, cveId int64) error { + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/hooks/" + hookId + push_events := "true" + tag_push_events := "true" + issues_events := "true" + note_events := "true" + merge_requests_events := "true" + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "url": "%s", + "password": "%s", + "push_events": "%s", + "tag_push_events": "%s", + "issues_events": "%s", + "note_events": "%s", + "merge_requests_events": "%s" + }`, accessToken, hookurl, pwd, push_events, tag_push_events, issues_events, note_events, merge_requests_events) + logs.Info("hook_body: ", requestBody) + resp, err := util.HTTPPatch(url, requestBody) + if err != nil { + logs.Error("更新钩子失败, url: ", url, ", err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("更新仓库 hook失败, err: ", ok, "url: ", url) + return errors.New("更新仓库 hook失败") + } + if resp["password"].(string) == pwd { + var issHook models.IssueHooks + CreateIssueHookData(&issHook, resp, + path, owner, issueNum, issueId, cveId) + hookId, err := models.CreateDepositHooks(&issHook) + if err != nil { + logs.Error("更新仓库 hook数据失败, repo: ", path, ", err: ", err) + return nil + } else { + logs.Info("更新仓库 hook数据成功, hookId: ", hookId, "repo: ", path) + } + } + return nil + } + return errors.New("更新仓库 hook失败") +} + +func PostWebHooks(accessToken, owner, path, hookurl, pwd, issueNum string, issueId, cveId int64) error { + if accessToken != "" && owner != "" && path != "" { + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/hooks" + push_events := "true" + tag_push_events := "true" + issues_events := "true" + note_events := "true" + merge_requests_events := "true" + requestBody := fmt.Sprintf(`{ + "access_token": "%s", + "url": "%s", + "password": "%s", + "push_events": "%s", + "tag_push_events": "%s", + "issues_events": "%s", + "note_events": "%s", + "merge_requests_events": "%s" + }`, accessToken, hookurl, pwd, push_events, tag_push_events, issues_events, note_events, merge_requests_events) + logs.Info("hook_body: ", requestBody) + resp, err := util.HTTPPost(url, requestBody) + if err != nil { + logs.Error("创建钩子失败, url: ", url, ", cveId:", cveId, ",err: ", err) + return err + } + if _, ok := resp["id"]; !ok { + logs.Error("创建仓库hook失败, err: ", ok, "url: ", url) + return errors.New("创建仓库hook失败") + } + if resp["password"].(string) == pwd { + var issHook models.IssueHooks + CreateIssueHookData(&issHook, resp, + path, owner, issueNum, issueId, cveId) + // Store issue data + hookId, err := models.CreateDepositHooks(&issHook) + if err != nil { + logs.Error("创建仓库hook失败, path: ", path, "err: ", err) + return err + } else { + logs.Info("创建仓库 hook数据成功, hookId: ", hookId, "path: ", path) + } + } + return nil + } + return errors.New("创建仓库hook失败") +} + func CreateDepositHooks(accessToken string, owner string, path string, cve models.VulnCenter, issueNum string, issueId int64) error { var ih models.IssueHooks @@ -454,131 +545,28 @@ func CreateDepositHooks(accessToken string, owner string, path string, if wh.HookUrl == hookurl { hookExist = true hookId := strconv.FormatInt(wh.HookId, 10) - if accessToken != "" && owner != "" && path != "" { - url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/hooks/" + hookId - push_events := "true" - tag_push_events := "true" - issues_events := "true" - note_events := "true" - merge_requests_events := "true" - requestBody := fmt.Sprintf(`{ - "access_token": "%s", - "url": "%s", - "password": "%s", - "push_events": "%s", - "tag_push_events": "%s", - "issues_events": "%s", - "note_events": "%s", - "merge_requests_events": "%s" - }`, accessToken, hookurl, pwd, push_events, tag_push_events, issues_events, note_events, merge_requests_events) - logs.Info("hook_body: ", requestBody) - resp, err := util.HTTPPatch(url, requestBody) - if err != nil { - logs.Error("创建钩子失败, url: ", url, "cveId", cve.CveId, "err: ", err) - return err - } - if _, ok := resp["id"]; !ok { - logs.Error("创建仓库 hook失败, err: ", ok, "url: ", url) - return errors.New("创建仓库 hook失败") - } - if resp["password"].(string) == pwd { - var issHook models.IssueHooks - CreateIssueHookData(&issHook, cve, resp, - path, owner, issueNum, issueId) - hookId, err := models.CreateDepositHooks(&issHook) - if err != nil { - logs.Error("创建仓库 hook数据失败, cveNum: ", cve.CveNum, "err: ", err) - return nil - } else { - logs.Info("创建仓库 hook数据成功, hookId: ", hookId, "cveNum: ", cve.CveNum) - } - } + patchErr := PatchWebHooks(accessToken, owner, path, hookurl, pwd, issueNum, hookId, issueId, cve.CveId) + if patchErr == nil { + logs.Info("owner: ", owner, ",repo: ", path, ",webhook updated successfully") + } else { + logs.Error("owner: ", owner, ",repo: ", path, ",patchErr: ", patchErr) } } } if !hookExist { - if accessToken != "" && owner != "" && path != "" { - url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/hooks" - push_events := "true" - tag_push_events := "true" - issues_events := "true" - note_events := "true" - merge_requests_events := "true" - requestBody := fmt.Sprintf(`{ - "access_token": "%s", - "url": "%s", - "password": "%s", - "push_events": "%s", - "tag_push_events": "%s", - "issues_events": "%s", - "note_events": "%s", - "merge_requests_events": "%s" - }`, accessToken, hookurl, pwd, push_events, tag_push_events, issues_events, note_events, merge_requests_events) - logs.Info("hook_body: ", requestBody) - resp, err := util.HTTPPost(url, requestBody) - if err != nil { - logs.Error("创建钩子失败, url: ", url, "cveId", cve.CveId, "err: ", err) - return err - } - if _, ok := resp["id"]; !ok { - logs.Error("创建仓库hook失败, err: ", ok, "url: ", url) - return errors.New("创建仓库hook失败") - } - if resp["password"].(string) == pwd { - var issHook models.IssueHooks - CreateIssueHookData(&issHook, cve, resp, - path, owner, issueNum, issueId) - // Store issue data - hookId, err := models.CreateDepositHooks(&issHook) - if err != nil { - logs.Error("创建仓库hook失败, cveNum: ", cve.CveNum, "err: ", err) - return err - } else { - logs.Info("创建仓库 hook数据成功, hookId: ", hookId, "cveNum: ", cve.CveNum) - } - } + postErr := PostWebHooks(accessToken, owner, path, hookurl, pwd, issueNum, issueId, cve.CveId) + if postErr == nil { + logs.Info("owner: ", owner, ",repo: ", path, ",webhook created successfully") + } else { + logs.Error("owner: ", owner, ",repo: ", path, ",postErr: ", postErr) } } } else { - if accessToken != "" && owner != "" && path != "" { - url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/hooks" - push_events := "true" - tag_push_events := "true" - issues_events := "true" - note_events := "true" - merge_requests_events := "true" - requestBody := fmt.Sprintf(`{ - "access_token": "%s", - "url": "%s", - "password": "%s", - "push_events": "%s", - "tag_push_events": "%s", - "issues_events": "%s", - "note_events": "%s", - "merge_requests_events": "%s" - }`, accessToken, hookurl, pwd, push_events, tag_push_events, issues_events, note_events, merge_requests_events) - logs.Info("hook_body: ", requestBody) - resp, err := util.HTTPPost(url, requestBody) - if err != nil { - logs.Error("创建钩子失败, url: ", url, "cveId", cve.CveId, "err: ", err) - return err - } - if _, ok := resp["id"]; !ok { - logs.Error("创建仓库hook失败, err: ", ok, "url: ", url) - return errors.New("创建仓库hook失败") - } - if resp["password"].(string) == pwd { - var issHook models.IssueHooks - CreateIssueHookData(&issHook, cve, resp, - path, owner, issueNum, issueId) - // Store issue data - hookID, err := models.CreateDepositHooks(&issHook) - if err != nil { - logs.Error("创建仓库hook失败, cveNum: ", cve.CveNum, "err: ", err) - return err - } - logs.Info("创建仓库 hook数据成功, hookID: ", hookID, "cveNum: ", cve.CveNum) - } + postErr := PostWebHooks(accessToken, owner, path, hookurl, pwd, issueNum, issueId, cve.CveId) + if postErr == nil { + logs.Info("owner: ", owner, ",repo: ", path, ",webhook created successfully") + } else { + logs.Error("owner: ", owner, ",repo: ", path, ", postErr: ", postErr) } } return nil diff --git a/taskhandler/grabissue.go b/taskhandler/grabissue.go index be7423d..83e1426 100644 --- a/taskhandler/grabissue.go +++ b/taskhandler/grabissue.go @@ -1,6 +1,7 @@ package taskhandler import ( + "cvevulner/common" "cvevulner/models" "cvevulner/util" "encoding/json" @@ -83,9 +84,9 @@ func GrabIssueByOrg(accToken, org string) error { //GrabIssueByRepo grab issue by repository func GrabIssueByRepo(accToken, owner, repo, state string) { page := 1 - product, err := getInfProduct(accToken, owner, repo) + product, err := GetInfProduct(accToken, owner, repo) if err != nil { - logs.Error("getInfProduct, err: ", err) + logs.Error("GetInfProduct, err: ", err) } desc := GetRepoDescription(repo) for { @@ -102,7 +103,8 @@ func GrabIssueByRepo(accToken, owner, repo, state string) { } } -func getInfProduct(token string, owner string, repo string) (infPro string, err error) { +func GetInfProduct(token string, owner string, repo string) (infPro string, err error) { + defer common.Catchs() resp, err := http.Get(fmt.Sprintf(GiteRepoBranch, owner, repo, token)) if err != nil { logs.Error("url: ", GiteRepoBranch, ",repo:", repo, ",err: ", err) @@ -177,7 +179,7 @@ func handleIssueList(list []models.HookIssue, product, desc string) { vb := strings.ReplaceAll(v.Body, "：", ":") item.Body = vb item.IssueExistTpl = models.IssueExistByNumber(v.Number, v.Id) - err := item.InsertOrUpdate() + err := item.InsertOrUpdate(1) if err != nil { logs.Error("insert or update issue fail:", err) } diff --git a/taskhandler/hook.go b/taskhandler/hook.go index e75c86c..6f8b434 100644 --- a/taskhandler/hook.go +++ b/taskhandler/hook.go @@ -3,10 +3,17 @@ package taskhandler import ( "cvevulner/models" "cvevulner/util" + "encoding/json" + "errors" "fmt" "github.com/astaxie/beego/logs" + "io/ioutil" + "net/http" + "sync" ) +var wgx sync.WaitGroup + type HookData struct { Id int64 HookUrl string @@ -78,3 +85,107 @@ func DeleteDepositHooks(accessToken, owner, repo string, hookId int64) { }`, accessToken) util.HTTPDelCom(url, requestBody) } + +func AddRepoWebhook(accessToken, owner, path, pwd, hookurl string) error { + var ih models.IssueHooks + ih.CveId = int64(1) + issueId := int64(1) + ih.IssueNum = "000000" + ih.Owner = owner + ih.Status = 1 + ih.Repo = path + ihs, errh := models.GetIssueHook(&ih) + if errh { + hookExist := false + for _, wh := range ihs { + if wh.HookUrl == hookurl { + hookExist = true + break + } + } + if !hookExist { + postErr := PostWebHooks(accessToken, owner, path, hookurl, pwd, ih.IssueNum, issueId, ih.CveId) + if postErr == nil { + logs.Info("owner: ", owner, ",repo: ", path, ",webhook created successfully") + } else { + logs.Error("owner: ", owner, ",repo: ", path, ", postErr: ", postErr) + } + return postErr + } + return nil + } else { + postErr := PostWebHooks(accessToken, owner, path, hookurl, pwd, ih.IssueNum, issueId, ih.CveId) + if postErr == nil { + logs.Info("owner: ", owner, ",repo: ", path, ",webhook created successfully") + } else { + logs.Error("owner: ", owner, ",repo: ", path, ", postErr: ", postErr) + } + return postErr + } +} + +//GetOrgRepos get organization repository +func GetOrgAllRepos(accToken, org, pwd, hookurl string, page int64) { + wgx.Add(1) + defer wgx.Done() + resp, err := http.Get(fmt.Sprintf(GiteOrgReposURL, org, accToken, page, perPage)) + if err != nil { + logs.Error("Get, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) + return + } + defer resp.Body.Close() + var reps []models.HookRepository + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error("ReadAll, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) + return + } + err = json.Unmarshal(body, &reps) + if err != nil { + logs.Error("Unmarshal, GiteOrgReposURL: ", GiteOrgReposURL, ", org: ", GiteOrgReposURL, ",err: ", err) + return + } + for _, v := range reps { + AddRepoWebhook(accToken, org, v.Name, pwd, hookurl) + } +} + +func GetAllRepoInfo(accessToken, owner, pwd, hookurl string) error { + orgInfo, err := GetOrgInfo(accessToken, owner) + if err != nil { + logs.Error("GetOrgInfo, org: ", owner, ",err: ", err) + return err + } + reposNum := orgInfo.PublicRepos + orgInfo.PrivateRepos + if reposNum <= 0 { + logs.Info(fmt.Sprintf("%v contain %v repository,grab issue finish!", owner, reposNum)) + return errors.New(fmt.Sprintf("%v contain %v repository,grab issue finish!", owner, reposNum)) + } + pageSize := reposNum / int64(perPage) + if reposNum%int64(perPage) > 0 { + pageSize = pageSize + 1 + } + var i int64 + for i = 1; i <= pageSize; i++ { + go GetOrgAllRepos(accessToken, owner, pwd, hookurl, i) + } + wgx.Wait() + return nil +} + +// Create webhook event function entry +func CreateHookEvent(hookurl, owner, accessToken, pwd, gaussOwner, gitGaussToken string) error { + repoErr := GetAllRepoInfo(accessToken, owner, pwd, hookurl) + if repoErr == nil { + logs.Info("CreateHookEvent, owner: ", owner, ", webhook created successfully, hookurl: ", hookurl) + } else { + logs.Error("CreateHookEvent, owner: ", owner, ", hookurl: ", hookurl, ",repoErr: ", repoErr) + } + dbRepo := models.QueryOrgAllRepo(owner) + if len(dbRepo) > 0 { + for _, v := range dbRepo { + AddRepoWebhook(accessToken, owner, v.Path, pwd, hookurl) + } + } + return nil +} -- Gitee