diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index 39bdf58a82f2fabf02e0fa6ae789744400835b9e..7562bc93d1262d3bdc736f6f4bf7423865cfec2e 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -2283,8 +2283,8 @@ func gitAddIssueProc(issueHook *models.IssuePayload, organizationID int8) error if err != nil { logs.Error("GetInfProduct, err: ", err) } - desc := taskhandler.GetRepoDescription(path) - giErr := AddGitIssue(issueHook, desc, product) + //desc := taskhandler.GetRepoDescription(path) + giErr := AddGitIssue(issueHook, "", product) if giErr != nil { logs.Error("AddGitIssue, giErr: ", giErr) } diff --git a/cve-vulner-manager/models/uploadcve.go b/cve-vulner-manager/models/uploadcve.go index 56bce479a8e28f768e0b82f02a0e8857dd60a672..fa91f213fb8736044ceee05a7d3f500b83769f91 100644 --- a/cve-vulner-manager/models/uploadcve.go +++ b/cve-vulner-manager/models/uploadcve.go @@ -1085,6 +1085,7 @@ func QueryCveOrigin(cveNum, typ string) bool { if err != nil { return false } + return true } @@ -1099,6 +1100,7 @@ func FilterOldData(cve string) bool { if res.Credibility > 3 && res.CveId <= 20557 { return true } + return false } @@ -1116,51 +1118,44 @@ func QueryCveErrorInfo(issueStatus int8, startDate, endDate string) ([]IssueTemp "where status > ? and create_time <= ?", issueStatus, endDate).QueryRows(&it) } if err != nil || num == 0 { - logs.Info("QueryCveErrorInfo, ", issueStatus, startDate, endDate, + logs.Error("QueryCveErrorInfo, ", issueStatus, startDate, endDate, ", cve_issue_template, No corresponding data") return it, false - } else { - logs.Info("QueryCveErrorInfo, ", issueStatus, startDate, endDate, - ", cve_issue_template, search successful") - return it, true } + + return it, true } -func GetOpengaussYaml(opy *OpenGussYaml, colName ...string) error { +func GetOpengaussYaml(opy *OpenGussYaml, colName ...string) (err error) { o := orm.NewOrm() - packageName := opy.PackageName + packageName := "%" + opy.PackageName + "%" if len(opy.PackageName) < 1 && len(opy.Version) < 1 { logs.Error("openGauss request query parameters: ", *opy) return errors.New("openGauss request query parameters error") } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { - err := o.Raw("select * from cve_open_guss_yaml where package_name = ? and version = ? "+ - "order by id asc limit ?", opy.PackageName, opy.Version, 1).QueryRow(opy) - if err != nil { - opy.Repo = packageName - opy.Id = 1 - logs.Error("GetOpengaussYaml1, err: ", err, *opy) - } - return err + err = o.Raw( + "select * from cve_open_guss_yaml where package_name like ? and version = ? "+ + "order by id asc limit ?", packageName, opy.Version, 1, + ).QueryRow(opy) } else { - err := o.Raw("select * from cve_open_guss_yaml where package_name = ? "+ - "order by id asc limit ?", opy.PackageName, 1).QueryRow(opy) - if err != nil { - logs.Error("GetOpengaussYaml2, err: ", err, *opy) - opy.Repo = packageName - opy.Id = 1 - } - return err + err = o.Raw( + "select * from cve_open_guss_yaml where package_name like ? order by id asc limit ?", packageName, 1, + ).QueryRow(opy) } + + return } func GetOpengaussYamlAll(opy *OpenGussYaml) (msy []OpenGussYaml, err error) { o := orm.NewOrm() - packageName := opy.PackageName + packageName := "%" + opy.PackageName + "%" if len(opy.PackageName) < 1 && len(opy.Version) < 1 { logs.Error("GetOpengaussYamlAll, openGauss request query parameters: ", *opy) return msy, errors.New("openGauss request query parameters error") } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { verStr := "(" verList := strings.Split(opy.Version, ",") @@ -1168,70 +1163,58 @@ func GetOpengaussYamlAll(opy *OpenGussYaml) (msy []OpenGussYaml, err error) { verStr += "'" + ver + "'," } verStr = verStr[:len(verStr)-1] + ")" - sql := fmt.Sprintf(`select * from cve_open_guss_yaml where package_name = '%s' -and version in %s group by repo_name order by id asc`, opy.PackageName, verStr) - num, err := o.Raw(sql).QueryRows(&msy) - if num == 0 { - opy.Repo = packageName - opy.Id = 1 - logs.Error("GetOpengaussYamlAll, err: ", err, *opy) - } - return msy, err + sql := fmt.Sprintf(`select * from cve_open_guss_yaml where package_name like '%s' +and version in %s group by repo_name order by id asc`, packageName, verStr) + _, err = o.Raw(sql).QueryRows(&msy) } else { - num, err := o.Raw("select * from cve_open_guss_yaml where package_name = ? "+ - "order by id asc", opy.PackageName).QueryRows(&msy) - if num == 0 { - logs.Error("GetOpengaussYamlAll2, err: ", err, *opy) - opy.Repo = packageName - opy.Id = 1 - } - return msy, err + _, err = o.Raw("select * from cve_open_guss_yaml where package_name like ? "+ + "order by id asc", packageName).QueryRows(&msy) } + + return } -func GetMindSporeYaml(opy *MindSporeYaml, colName ...string) error { +func GetMindSporeYaml(opy *MindSporeYaml, colName ...string) (err error) { o := orm.NewOrm() - packageName := opy.PackageName if len(opy.PackageName) < 1 && len(opy.Version) < 1 { logs.Error("Mindspore request query parameters: ", *opy) return errors.New("Mindspore request query parameters error") } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { - err := o.Raw("select * from cve_mind_spore_yaml where package_name = ? and version = ? "+ - "order by id asc limit ?", opy.PackageName, opy.Version, 1).QueryRow(opy) - if err != nil { - opy.Repo = packageName - opy.Id = 1 - logs.Error("GetMindSporeYaml1, err: ", err, *opy) - } - return err + err = o.Raw( + "select * from cve_mind_spore_yaml where package_name = ? and version = ? "+ + "order by id asc limit ?", opy.PackageName, opy.Version, 1, + ).QueryRow(opy) + return } else { - err := o.Raw("select * from cve_mind_spore_yaml where package_name = ? "+ - "order by id asc limit ?", opy.PackageName, 1).QueryRow(opy) - if err != nil { - logs.Error("GetMindSporeYaml2, err: ", err, *opy) - opy.Repo = packageName - opy.Id = 1 - } - return err + err = o.Raw( + "select * from cve_mind_spore_yaml where package_name = ? order by id asc limit ?", opy.PackageName, 1, + ).QueryRow(opy) } + + return } + func GetMindSporeYamlForids(ids []string) (msy []MindSporeYaml, err error) { if len(ids) == 0 { return } o := orm.NewOrm() - _, err = o.Raw(fmt.Sprintf("select * from cve_mind_spore_yaml where id in (%s) order by id asc", strings.Join(ids, ","))).QueryRows(&msy) + _, err = o.Raw( + fmt.Sprintf("select * from cve_mind_spore_yaml where id in (%s) order by id asc", strings.Join(ids, ",")), + ).QueryRows(&msy) + return } func GetMindSporeYamlAll(opy *MindSporeYaml) (msy []MindSporeYaml, err error) { o := orm.NewOrm() - packageName := opy.PackageName if len(opy.PackageName) < 1 && len(opy.Version) < 1 { logs.Error("GetMindSporeYamlAll, Mindspore request query parameters: ", *opy) return msy, errors.New("Mindspore request query parameters error") } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { verStr := "(" verList := strings.Split(opy.Version, ",") @@ -1239,75 +1222,63 @@ func GetMindSporeYamlAll(opy *MindSporeYaml) (msy []MindSporeYaml, err error) { verStr += "'" + ver + "'," } verStr = verStr[:len(verStr)-1] + ")" - sql := fmt.Sprintf(`select * from cve_mind_spore_yaml where package_name = '%s' -and version in %s group by repo_name order by id asc`, opy.PackageName, verStr) - num, err := o.Raw(sql).QueryRows(&msy) - if num == 0 { - opy.Repo = packageName - opy.Id = 1 - logs.Error("GetMindSporeYamlAll, err: ", err, *opy) - } - return msy, err + sql := fmt.Sprintf( + `select * from cve_mind_spore_yaml where package_name = '%s' and version in %s group by repo_name order by id asc`, + opy.PackageName, verStr) + _, err = o.Raw(sql).QueryRows(&msy) } else { - num, err := o.Raw("select * from cve_mind_spore_yaml where package_name = ? "+ + _, err = o.Raw("select * from cve_mind_spore_yaml where package_name = ? "+ "order by id asc", opy.PackageName).QueryRows(&msy) - if num == 0 { - logs.Error("GetMindSporeYamlAll2, err: ", err, *opy) - opy.Repo = packageName - opy.Id = 1 - } - return msy, err } + + return } func InsertOriginCveRecord(our *OriginUpstreamRecord) error { o := orm.NewOrm() - id, err := o.Insert(our) - logs.Info("InsertOriginCveRecord, id: ", id, ", err: ", err) + _, err := o.Insert(our) + return err } func DeleteOriginCveRecord(beforeDate string) { o := orm.NewOrm() - err := o.Raw("delete from cve_origin_upstream_record where create_time < ?", beforeDate).QueryRow() - logs.Info("DeleteOriginCveRecord", err) + if err := o.Raw( + "delete from cve_origin_upstream_record where create_time < ?", beforeDate, + ).QueryRow(); err != nil { + logs.Info("DeleteOriginCveRecord", err) + } } -func GetOpenLookengYaml(opy *OpenLookengYaml, colName ...string) error { +func GetOpenLookengYaml(opy *OpenLookengYaml, colName ...string) (err error) { o := orm.NewOrm() - packageName := opy.PackageName if len(opy.PackageName) < 1 && len(opy.Version) < 1 { logs.Error("OpenLooKeng request query parameters: ", *opy) return errors.New("OpenLooKeng request query parameters error") } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { - err := o.Raw("select * from cve_open_lookeng_yaml where package_name = ? and version = ? "+ - "order by id asc limit ?", opy.PackageName, opy.Version, 1).QueryRow(opy) - if err != nil { - opy.Repo = packageName - opy.Id = 1 - logs.Error("GetOpenLookengYaml1, err: ", err, *opy) - } - return err + err = o.Raw( + "select * from cve_open_lookeng_yaml where package_name = ? and version = ? "+ + "order by id asc limit ?", opy.PackageName, opy.Version, 1, + ).QueryRow(opy) + } else { - err := o.Raw("select * from cve_open_lookeng_yaml where package_name = ? "+ - "order by id asc limit ?", opy.PackageName, 1).QueryRow(opy) - if err != nil { - logs.Error("GetOpenLookengYaml12, err: ", err, *opy) - opy.Repo = packageName - opy.Id = 1 - } - return err + err = o.Raw( + "select * from cve_open_lookeng_yaml where package_name = ? order by id asc limit ?", opy.PackageName, 1, + ).QueryRow(opy) } + + return } func GetOpenLookengYamlAll(opy *OpenLookengYaml) (msy []OpenLookengYaml, err error) { o := orm.NewOrm() - packageName := opy.PackageName if len(opy.PackageName) < 1 && len(opy.Version) < 1 { logs.Error("GetOpenLookengYamlAll, openLookeng request query parameters: ", *opy) return msy, errors.New("openLookeng request query parameters error") } + if len(opy.PackageName) > 1 && len(opy.Version) > 0 { verStr := "(" verList := strings.Split(opy.Version, ",") @@ -1317,21 +1288,12 @@ func GetOpenLookengYamlAll(opy *OpenLookengYaml) (msy []OpenLookengYaml, err err verStr = verStr[:len(verStr)-1] + ")" sql := fmt.Sprintf(`select * from cve_open_lookeng_yaml where package_name = '%s' and version in %s group by repo_name order by id asc`, opy.PackageName, verStr) - num, err := o.Raw(sql).QueryRows(&msy) - if num == 0 { - opy.Repo = packageName - opy.Id = 1 - logs.Error("GetOpenLookengYamlAll, err: ", err, *opy) - } - return msy, err + _, err = o.Raw(sql).QueryRows(&msy) } else { - num, err := o.Raw("select * from cve_open_lookeng_yaml where package_name = ? "+ - "order by id asc", opy.PackageName).QueryRows(&msy) - if num == 0 { - logs.Error("GetOpenLookengYamlAll2, err: ", err, *opy) - opy.Repo = packageName - opy.Id = 1 - } - return msy, err + _, err = o.Raw( + "select * from cve_open_lookeng_yaml where package_name = ? order by id asc", opy.PackageName, + ).QueryRows(&msy) } + + return } diff --git a/cve-vulner-manager/taskhandler/common.go b/cve-vulner-manager/taskhandler/common.go index 94debbd082e7c0412f4e1c8b20ebb3dd44147ea3..982811664d5d741269eeff3b18b1bf44ae0f4637 100644 --- a/cve-vulner-manager/taskhandler/common.go +++ b/cve-vulner-manager/taskhandler/common.go @@ -28,7 +28,7 @@ const ( GiteRepoBranch = `https://gitee.com/api/v5/repos/%v/%v/branches?access_token=%v` //RepoInfoURL get repo info url RepoInfoURL = "https://api.openeuler.org/pkgmanage/packages/packageInfo?table_name=openEuler_LTS_20.03&pkg_name=%s" - perPage = 50 + perPage = 100 //IssueType Types of issues crawled CIssueType = "CVE和安全问题" GaussIssueType = "缺陷" diff --git a/cve-vulner-manager/taskhandler/grabissue.go b/cve-vulner-manager/taskhandler/grabissue.go index ade0f9a0ac4292db035e20fb65b0f501a6561add..44d92798343a66833a2a14fb0dc96eeb0492bf75 100644 --- a/cve-vulner-manager/taskhandler/grabissue.go +++ b/cve-vulner-manager/taskhandler/grabissue.go @@ -1,18 +1,21 @@ package taskhandler import ( - "cvevulner/common" - "cvevulner/models" - "cvevulner/util" "encoding/json" "errors" "fmt" - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" "io/ioutil" "net/http" "strings" "sync" + "time" + + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + + "cvevulner/common" + "cvevulner/models" + "cvevulner/util" ) var wg sync.WaitGroup @@ -62,21 +65,27 @@ func GrabIssueByOrg(accToken, org string) error { logs.Error("GetOrgInfo, org: ", org, ",err: ", err) return err } + reposNum := orgInfo.PublicRepos + orgInfo.PrivateRepos if reposNum <= 0 { logs.Info(fmt.Sprintf("%v contain %v repository,grab issue finish!", org, reposNum)) return errors.New(fmt.Sprintf("%v contain %v repository,grab issue finish!", org, reposNum)) } + pageSize := reposNum / int64(perPage) if reposNum%int64(perPage) > 0 { pageSize = pageSize + 1 } + var i int64 for i = 1; i <= pageSize; i++ { + time.Sleep(time.Second * 20) go GetOrgRepos(accToken, org, i) } + wg.Wait() logs.Info("Synchronize gitee's issue finish...") + return nil } @@ -87,14 +96,14 @@ func GrabIssueByRepo(accToken, owner, repo, state string) { if err != nil { logs.Error("GetInfProduct, err: ", err) } - desc := GetRepoDescription(repo) + //desc := GetRepoDescription(repo) for { list, err := GetIssueList(accToken, owner, repo, state, page) if err != nil { logs.Error("GetIssueList, repo: ", repo, ",err: ", err) break } - handleIssueList(list, product, desc) + handleIssueList(list, product, "") if len(list) < perPage { break } @@ -195,11 +204,24 @@ func handleIssueList(list []models.GitIssue, product, desc string) { //GetOrgInfo get organization information func GetOrgInfo(accToken, org string) (OrgInfo, error) { oi := OrgInfo{} - resp, err := http.Get(fmt.Sprintf(GiteOrgInfoURL, org, accToken)) - if err != nil { - return oi, err + var ( + resp *http.Response + err error + ) + + for i := 0; i < 4; i++ { + resp, err = http.Get(fmt.Sprintf(GiteOrgInfoURL, org, accToken)) + if err == nil { + break + } else if err != nil && i == 3 { + return OrgInfo{}, err + } + + time.Sleep(time.Second * 3) } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) if err != nil { return oi, err