diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index 47c266e1ba38c120c507136f5767a5d5cdeca100..9f9e8017afe3b86861b5808f072e05b23eab7146 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -96,6 +96,8 @@ const ( cmdCheckIssue = "/check-issue" reasonCommand = "/reason" reasonX = "xxxxxx" + + openEulerBotName = "openeuler-ci-bot" ) var comLock sync.Mutex @@ -232,6 +234,10 @@ func (c *HookEventControllers) handleIssue() { } } if issueHook.Action == "open" { + if issueHook.User.Login == openEulerBotName { + return + } + owner, token := common.GetOwnerAndToken("", organizationID) issueTmp := models.IssueTemplate{IssueNum: issueHook.Iid, IssueId: issueHook.Issue.Id} err = models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") diff --git a/cve-vulner-manager/cve-timed-task/tabletask/packages.go b/cve-vulner-manager/cve-timed-task/tabletask/packages.go index f4f43eb64f04617efdb265dfe45f704deea034fe..e168d2725224e6e28de36b73fc8a7c55b3b41d0e 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/packages.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/packages.go @@ -101,7 +101,7 @@ type response struct { } func getPackageFromNG(page int) ([]Packages, error) { - url := fmt.Sprintf("%s/api/v1/packages?page=%d&per_page=%d", + url := fmt.Sprintf("%s/api/v1/packages?page=%d&per_page=%d&exclude_org=ascend", beego.AppConfig.String("cve-manager-ng::host"), page, pageSize) const maxRetry = 3 diff --git a/cve-vulner-manager/models/ymal.go b/cve-vulner-manager/models/ymal.go index 1b9bca127f402cdc601b63b04d5f5448ab345a9b..5bb3e3c6d7a0225c9098ff6c277ab455b720a8a2 100644 --- a/cve-vulner-manager/models/ymal.go +++ b/cve-vulner-manager/models/ymal.go @@ -5,277 +5,6 @@ import ( "github.com/astaxie/beego/orm" ) -func GetYamlTable(gt *[]GitPackageTable) (int64, error) { - o := orm.NewOrm() - num, errx := o.Raw("select *" + - " from cve_git_package_table").QueryRows(gt) - if errx != nil { - logs.Error("GetYamlTable, Get data does not exist, err: ", errx) - } - return num, errx -} - -func GetCpePackName(pc *PackageCpe, packName string) (error) { - o := orm.NewOrm() - errx := o.Raw("select id, cpe_packname"+ - " from cve_package_cpe where packname = ?", packName).QueryRow(pc) - return errx -} - -func CreateYamlTable(gt *GitPackageTable) (TableId int64, err error) { - o := orm.NewOrm() - var localgt GitPackageTable - if gt.TableName == "openEuler_LTS_20.03" { - gt.Brands = "openEuler-20.03-LTS" - } else { - gt.Brands = "master" - } - errx := o.Raw("select *"+ - " from cve_git_package_table where table_name = ? ", gt.TableName).QueryRow(&localgt) - if errx != nil { - var TableId int64 - if TableId, err = o.Insert(gt); err == nil { - logs.Info("CreateYamlTable, insert cve_git_package_table success, "+ - "TableId: ", TableId, ", tableName: ", gt.TableName) - } else { - logs.Error("CreateYamlTable, insert cve_git_package_table failed,"+ - " tableName:", gt.TableName, ", err: ", err) - return 0, err - } - return TableId, nil - } else { - gt.TableId = localgt.TableId - gt.Brands = localgt.Brands - if num, err := o.Update(gt); err == nil { - logs.Info("CreateYamlTable, update cve_git_package_table success, "+ - "num: ", num, ", TableName: ", gt.TableName) - } else { - logs.Error("CreateYamlTable, update cve_git_package_table failed, "+ - "TableName:", gt.TableName, ", err: ", err) - return 0, err - } - return gt.TableId, nil - } -} - -func GetSingleYaml(ge *GitOpenEuler) (bool, error) { - o := orm.NewOrm() - if ge.Version != "" && ge.Release != "" { - localge := GitOpenEuler{PackageName: ge.PackageName, Version: ge.Version, Status: 1} - err := o.Read(&localge, "PackageName", "Version", "Release") - logs.Info("GetSingleYaml1, Query yaml data results: ", ge, - ", localge: ", localge, ", err: ", err) - if err == orm.ErrNoRows { - return false, err - } else if err == orm.ErrMissPK { - return false, err - } else { - return true, err - } - } else { - localge := GitOpenEuler{PackageName: ge.PackageName, Status: 1} - err := o.Read(&localge, "PackageName") - logs.Info("GetSingleYaml2, Query yaml data results: ", ge, - ", localge: ", localge, ", err: ", err) - if err == orm.ErrNoRows { - return false, err - } else if err == orm.ErrMissPK { - return false, err - } else { - return true, err - } - } -} - -func InsertOpenEulerRelate(getr *GitOpenEulerTableRelate) (id int64, typex string, err error) { - o := orm.NewOrm() - var localge GitOpenEulerTableRelate - errx := o.Raw("select *"+ - " from cve_git_open_euler_table_relate where git_id = ? and table_name = ?", - getr.GitId, getr.TableName).QueryRow(&localge) - if errx != nil { - var relateId int64 - if relateId, err = o.Insert(getr); err == nil { - logs.Info("InsertOpenEulerRelate, insert cve_git_open_euler_table_relate success, "+ - "RelateId: ", relateId, "tableName: ", getr.TableName) - } else { - logs.Error("InsertOpenEulerRelate, insert cve_git_open_euler_table_relate failed, "+ - "tableName: ", getr.TableName, ", err: ", err) - return 0, "insert", err - } - return relateId, "insert", nil - } else { - getr.RelateId = localge.RelateId - if _, err := o.Update(getr); err == nil { - logs.Info("InsertOpenEulerRelate, update cve_git_open_euler_table_relate success, "+ - "RelateId: ", getr.RelateId, "tableName: ", getr.TableName) - } else { - logs.Error("InsertOpenEulerRelate, update cve_git_open_euler_table_relate failed, "+ - "tableName: ", getr.TableName, ", err: ", err) - return getr.RelateId, "update", err - } - return getr.RelateId, "update", nil - } -} - -func CreateYaml(ge *GitOpenEuler, tableValue GitPackageTable) (id int64, typex string, err error) { - o := orm.NewOrm() - var localge GitOpenEuler - var getr GitOpenEulerTableRelate - getr.TableName = tableValue.TableName - getr.Brands = tableValue.Brands - errx := o.Raw("select *"+ - " from cve_git_open_euler where package_name = ? and version = ?", - ge.PackageName, ge.Version).QueryRow(&localge) - if errx != nil { - logs.Info("createYaml: ", localge, ",errx: ", errx) - ge.Status = 1 - var GitId int64 - if GitId, err = o.Insert(ge); err == nil { - logs.Info("CreateYaml, insert cve_git_open_euler success, GitId: ", - GitId, ", packName: ", ge.PackageName) - } else { - logs.Error("CreateYaml, insert cve_git_open_euler failed, "+ - "packName: ", ge.PackageName, ", err: ", err) - return 0, "insert", err - } - getr.GitId = GitId - InsertOpenEulerRelate(&getr) - return GitId, "insert", nil - } else { - logs.Info("createYaml: ", localge, "GitOpenEuler: ", ge) - ge.Status = 1 - ge.GitId = localge.GitId - if _, err := o.Update(ge); err == nil { - logs.Info("CreateYaml, update cve_git_open_euler success, "+ - "GitId: ", ge.GitId, ", packName: ", ge.PackageName) - } else { - logs.Error("CreateYaml, update cve_git_open_euler failed, "+ - "packName: ", ge.PackageName, ", err: ", err) - return ge.GitId, "update", err - } - getr.GitId = ge.GitId - InsertOpenEulerRelate(&getr) - return ge.GitId, "update", nil - } -} - -func CreateYamlDetail(gp *GitPackageInfo, ge GitOpenEuler) (id int64, typex string, err error) { - o := orm.NewOrm() - var localgp GitPackageInfo - errx := o.Raw("select *"+ - " from cve_git_package_info where git_id = ? and package_name = ? and version = ?", - ge.GitId, gp.PackageName, gp.Version).QueryRow(&localgp) - if errx != nil { - gp.Status = 0 - var DetailId int64 - if DetailId, err = o.Insert(gp); err == nil { - logs.Info("CreateYamlDetail, insert cve_git_package_info success, "+ - "DetailId: ", DetailId, ",PackageName: ", gp.PackageName) - } else { - logs.Error("CreateYamlDetail, insert cve_git_package_info failed, "+ - "PackageName: ", gp.PackageName, ",err: ", err) - return 0, "insert", err - } - return DetailId, "insert", nil - } else { - gp.DetailId = localgp.DetailId - if _, err := o.Update(gp); err == nil { - logs.Info("CreateYamlDetail, update cve_git_package_info success, "+ - "DetailId: ", gp.DetailId, ",packName: ", gp.PackageName) - } else { - logs.Error("CreateYamlDetail, update cve_git_package_info failed, "+ - "packName: ", gp.PackageName, ",err: ", err) - return gp.DetailId, "update", err - } - return gp.DetailId, "update", nil - } -} - -func CreateYamlSubPack(gb *GitSubPack) (SubId int64, typex string, err error) { - o := orm.NewOrm() - var localgb GitSubPack - errx := o.Raw("select *"+ - " from cve_git_sub_pack where detail_id = ? and ids = ? and sub_pack_name = ?", - gb.DetailId, gb.Ids, gb.SubPackName).QueryRow(&localgb) - if errx != nil { - var SubId int64 - if SubId, err = o.Insert(gb); err == nil { - logs.Info("CreateYamlSubPack, insert cve_git_sub_pack success,"+ - " SubId: ", SubId, ",SubPackName: ", gb.SubPackName) - } else { - logs.Error("CreateYamlSubPack, insert cve_git_sub_pack failed, "+ - "SubPackName: ", gb.SubPackName, ",err: ", err) - return 0, "insert", err - } - return SubId, "insert", nil - } - return localgb.SubId, "update", errx -} - -func CreateYamlSubPackProvides(gs *GitSubPackProvides) (ProvideId int64, typex string, err error) { - o := orm.NewOrm() - var localgs GitSubPackProvides - errx := o.Raw("select *"+ - " from cve_git_sub_pack_provides where sub_id = ? and ids = ? and provide_name = ?", - gs.SubId, gs.Ids, gs.ProvideName).QueryRow(&localgs) - if errx != nil { - var ProvideId int64 - if ProvideId, err = o.Insert(gs); err == nil { - logs.Info("CreateYamlSubPack, insert cve_git_sub_pack_provides success, "+ - "ProvideId: ", ProvideId, ", ProvideName: ", gs.ProvideName) - } else { - logs.Error("CreateYamlSubPack, insert cve_git_sub_pack_provides failed, "+ - "ProvideName: ", gs.ProvideName, ",err: ", err) - return 0, "insert", err - } - return ProvideId, "insert", nil - } - return localgs.ProvideId, "update", errx -} - -func CreateYamlSubPackRequiredb(gs *GitSubPackRequiredby) (Id int64, typex string, err error) { - o := orm.NewOrm() - var localgs GitSubPackRequiredby - errx := o.Raw("select *"+ - " from cve_git_sub_pack_requiredby where provide_id = ? and requiredb = ?", - gs.ProvideId, gs.Requiredby).QueryRow(&localgs) - if errx != nil { - var Id int64 - if Id, err = o.Insert(gs); err == nil { - logs.Info("CreateYamlSubPackRequiredb, insert cve_git_sub_pack_requiredby success, "+ - "ID: ", Id, ", requiredb: ", gs.Requiredby) - } else { - logs.Error("CreateYamlSubPackRequiredb, insert cve_git_sub_pack_requiredby failed, "+ - "requiredb: ", gs.Requiredby, ", err: ", err) - return 0, "insert", err - } - return Id, "insert", nil - } - return localgs.Id, "update", errx -} - -func CreateYamlSubPackRequires(gs *GitSubPackRequire) (RequireId int64, typex string, err error) { - o := orm.NewOrm() - var localgs GitSubPackRequire - errx := o.Raw("select *"+ - " from cve_git_sub_pack_require where sub_id = ? and ids = ? and require_name = ?", - gs.SubId, gs.Ids, gs.RequireName).QueryRow(&localgs) - if errx != nil { - var RequireId int64 - if RequireId, err = o.Insert(gs); err == nil { - logs.Info("CreateYamlSubPackRequires, insert cve_git_sub_pack_require success,"+ - " RequireId: ", RequireId, ",RequireName: ", gs.RequireName) - } else { - logs.Error("CreateYamlSubPackRequires, insert cve_git_sub_pack_require failed, "+ - "RequireName: ", gs.RequireName, ", err: ", err) - return 0, "insert", err - } - return RequireId, "insert", nil - } - return localgs.RequireId, "update", errx -} - func QueryEulerRepoOrigin(ogk *OpenEulerRepoOrigin, field ...string) error { o := orm.NewOrm() err := o.Read(ogk, field...) @@ -305,6 +34,27 @@ func UpdateEulerRepoOrigin(ogk *OpenEulerRepoOrigin, fields ...string) error { return err } +// DeleteEulerRepoOrigin delete all data +func DeleteEulerRepoOrigin() error { + o := orm.NewOrm() + _, err := o.Raw("delete from cve_open_euler_repo_origin").Exec() + return err +} + +// DeleteGitOpenEuler delete data of ng +func DeleteGitOpenEuler() error { + o := orm.NewOrm() + _, err := o.Raw("delete from cve_git_open_euler where source != ? ", "ng").Exec() + return err +} + +// DeleteGitPackageInfo delete all data +func DeleteGitPackageInfo() error { + o := orm.NewOrm() + _, err := o.Raw("delete from cve_git_package_info").Exec() + return err +} + func GetEulerYamlList(yamlId int64, count int) (oer []OpenEulerRepoOrigin) { o := orm.NewOrm() num, geErr := o.Raw("select *"+ @@ -325,15 +75,3 @@ func GetGitOpenEulerList(gitId int64, count int) (goe []GitOpenEuler) { } return } - -func DelOpenEulerRepoOriginByName(packageName string) { - o := orm.NewOrm() - _ = o.Raw("DELETE FROM cve_open_euler_repo_origin WHERE package_name = ?", packageName).QueryRow() - return -} - -func TruncateOpenEulerRepoOrigin() { - o := orm.NewOrm() - _ = o.Raw("TRUNCATE table cve_open_euler_repo_origin").QueryRow() - return -} diff --git a/cve-vulner-manager/task/yaml.go b/cve-vulner-manager/task/yaml.go index fd9261c8108106c216d46d338e4130242a650237..540290f3fe75fa9f72bda39dcdd2bf7b2dd59f78 100644 --- a/cve-vulner-manager/task/yaml.go +++ b/cve-vulner-manager/task/yaml.go @@ -1,30 +1,10 @@ package task import ( - "errors" - - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" - - "cvevulner/common" "cvevulner/taskhandler" ) // GetEulerYamlData Get yaml data source func GetEulerYamlData() error { - defer common.Catchs() - logs.Info("Get the euleryaml data source task started") - // Query the cve to be processed, 1: add; 2: modify - eulerUrl := beego.AppConfig.String("yaml::eulerurl") - if eulerUrl == "" { - logs.Error("GetEulerYamlData, config yaml::eulerUrl, error: invalid value") - return errors.New("eulerUrl value is nil") - } - _, err := taskhandler.GetEulerYamlInfo(eulerUrl) - // Synchronize other sources of yaml version - taskhandler.SyncEulerYaml() - // Delete historical yaml source version data that does not exist - taskhandler.DelHistoryEulerYaml() - logs.Info("End of the task of obtaining euleryaml data source") - return err + return taskhandler.SyncEulerPackages() } diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index 8136b222ccf23bb81d9ce286b2fd6adb6b1c8dc9..74947683429553f32bcb46cf67bc3c9036ee3d02 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -29,6 +29,7 @@ var ( pkgLock sync.Mutex lockx sync.Mutex dbLock sync.Mutex + lock sync.Mutex ) var ewg sync.WaitGroup diff --git a/cve-vulner-manager/taskhandler/issuestatistics.go b/cve-vulner-manager/taskhandler/issuestatistics.go index 91868d975b3dfb56c73870b9d5b63213af96d100..48ef1079548f43a014389d50a239fb5ddd538f36 100644 --- a/cve-vulner-manager/taskhandler/issuestatistics.go +++ b/cve-vulner-manager/taskhandler/issuestatistics.go @@ -243,9 +243,11 @@ func IssueStatistics(beforeMonth, prcnum int, owner string) error { } } zipFileName := excelPath + + const deci = 10 if is.cveCount > 0 { if templateId > 0 { - cBody := fmt.Sprintf("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, 10) + "个, 其中" + + cBody := fmt.Sprint("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, deci) + "个, 其中" + strconv.FormatInt(is.HighCveCount, 10) + "个7分以上漏洞, 详情见附件, 7分以上CVE请在周三之前解决. \r\n" + "已经分析完毕的issue请maintainer尽快关掉, 否则影响数据统计; 提交PR时要关联issue, 若CVE在之前PR解决, 请PR提交人编辑PR信息将issue关联上. \r\n") sendError := SendEmail(zipFileName, 2, cBody, "") diff --git a/cve-vulner-manager/taskhandler/sendemail.go b/cve-vulner-manager/taskhandler/sendemail.go index 2e3f3300920935dce81c480f216551008c9aec3a..218b8228cdfa26210b82e1ff80425715b9834519 100644 --- a/cve-vulner-manager/taskhandler/sendemail.go +++ b/cve-vulner-manager/taskhandler/sendemail.go @@ -3,13 +3,8 @@ package taskhandler import ( "archive/zip" "bytes" - "cvevulner/common" - "cvevulner/models" "encoding/base64" "fmt" - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" - "gopkg.in/gomail.v2" "io" "io/ioutil" "mime" @@ -19,6 +14,13 @@ import ( "strconv" "strings" "time" + + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "gopkg.in/gomail.v2" + + "cvevulner/common" + "cvevulner/models" ) // define email interface, and implemented auth and send method @@ -137,7 +139,7 @@ func SendEmail(attchStr string, flag int, cBody, subject string) error { //} // subject := "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime() - body := fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been " + + body := fmt.Sprint("hi all: \r\n The list of cvrf format files that have been " + "uploaded to the file server is as follows: \r\n" + strings.ReplaceAll(attchStr, "download/", "")) emailCc := []string{} sendErr := SendCommonMail(toEmailName, emailCc, subject, body, attchStr) @@ -368,7 +370,8 @@ func SendCvrfEmail(subFileSlice []string) { } subject := "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime() fileStr := strings.Join(subFileSlice, "\r\n") - body := fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been uploaded to the file server is as follows: \r\n" + fileStr) + body := fmt.Sprint("hi all: \r\n The list of cvrf format files that have been" + + " uploaded to the file server is as follows: \r\n" + fileStr) sendErr := SendCommonMail(toEmailName, []string{}, subject, body, "") if sendErr == nil { logs.Info("Notify cvrf that the email was sent successfully!") diff --git a/cve-vulner-manager/taskhandler/yaml.go b/cve-vulner-manager/taskhandler/yaml.go index ad46f4875da655f538c427bb3ab6e153ec0eaf40..1a1e64ac0934a87fa926b3d2b4927fbbd6a9816f 100644 --- a/cve-vulner-manager/taskhandler/yaml.go +++ b/cve-vulner-manager/taskhandler/yaml.go @@ -1,290 +1,219 @@ package taskhandler import ( - "encoding/json" "fmt" "net/http" "strings" - "sync" + + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "github.com/opensourceways/server-common-lib/utils" "cvevulner/common" "cvevulner/models" - "cvevulner/util" +) - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" +const ( + packageUrl = "https://magicapi.osinfra.cn/repo/versions" + pageSize = 1000 ) -var lock sync.Mutex +// SyncEulerPackages sync packages of openeuler +func SyncEulerPackages() error { + data, err := getEulerPackages() + if err != nil { + return err + } -func GetEulerYamlInfo(url string) (string, error) { - defer common.Catchs() - compUrl := url + "?community=openeuler" - body, err := util.HTTPGetCom(compUrl) - var respBody map[string]interface{} - if err == nil && body != nil { - err = json.Unmarshal(body, &respBody) - if err != nil { - logs.Error(err) - return "", err - } - } else { - logs.Error("http Request failed, url: ", compUrl) - return "", err + if err = cleanOldData(); err != nil { + return err } - data, ok := respBody["data"] - if !ok || len(data.([]interface{})) == 0 { - logs.Error("Data is empty, url: ", compUrl) - return "", fmt.Errorf("Data is empty, url: %s", compUrl) + handle := func(path string, detail []BranchDetail) { + originByVersion := groupByVersion(path, detail) + for _, origin := range originByVersion { + if err = saveData(origin); err != nil { + logs.Error("save data failed: ", err.Error()) + } + } } - code, ok2 := respBody["code"] - total, ok3 := respBody["total"] - if ok2 && ok3 && code.(float64) == http.StatusOK && total.(float64) > 0 { - //models.TruncateOpenEulerRepoOrigin() - outSyncBranch := beego.AppConfig.String("cve::out_of_sync_branch") - outSyncBranchList := strings.Split(outSyncBranch, ",") - for _, values := range data.([]interface{}) { - valuesX := values.(map[string]interface{}) - ProcYamlInfo(valuesX, outSyncBranchList) + for _, v := range data { + for k, detail := range v { + handle(k, detail) } } - return "", nil + + return nil } -// Parse the received json data -func ProcYamlInfo(valuesX map[string]interface{}, outSyncBranch []string) { - if valuesX != nil && len(valuesX) > 0 { - for repKey, repValue := range valuesX { - if repKey != "" && len(repKey) > 1 { - var branchs = make(map[string]string) - //Delete all data first, then insert new data - //models.DelOpenEulerRepoOriginByName(repKey) - value := repValue.(map[string]interface{}) - brValue, brOk := value["branch_detail"] - if brOk { - brKey := brValue.([]interface{}) - if brKey != nil && len(brKey) > 0 { - for _, branchValue := range brKey { - var er models.OpenEulerRepoOrigin - er.PackageName = repKey - v := branchValue.(map[string]interface{}) - var ver string - for k, vv := range v { - if strings.EqualFold(k, "version") { - ver = vv.(string) - if _, ok := branchs[ver]; !ok { - branchs[ver] = "" - } - } - } - for vKey, vu := range v { - switch vKey { - case "summary": - er.Summary = vu.(string) - case "version": - er.Version = vu.(string) - case "brname": - er.Branchs = vu.(string) - if !filterDataInSlice(er.Branchs, outSyncBranch) { - branch, ok := branchs[ver] - if !ok { - branchs[ver] = vu.(string) - } else { - if len(branch) == 0 { - branchs[ver] = vu.(string) - } else { - branchs[ver] = branch + "," + vu.(string) - } - } - } - case "description": - switch vu.(type) { - case []interface{}: - des := vu.([]interface{}) - for _, d := range des { - if er.Decription == "" { - er.Decription = d.(string) - } else { - er.Decription = er.Decription + "," + d.(string) - } - } - case string: - er.Decription = vu.(string) - } - } - } - if len(er.Branchs) > 0 { - outFlag := false - for _, outBranch := range outSyncBranch { - if outBranch == er.Branchs { - outFlag = true - break - } - } - if outFlag { - continue - } - } - UpdateYamlToDb(er, branchs) - } - } - } - } - } +func saveData(origin models.OpenEulerRepoOrigin) error { + if _, err := models.InsertEulerRepoOrigin(&origin); err != nil { + return err + } + + goeId, err := models.InsertYamlData(&models.GitOpenEuler{ + PackageName: origin.PackageName, + Version: origin.Version, + PackageId: 0, + TableId: 0, + Status: 1, + Source: "openeuler", + }) + if err != nil { + return err } - return + + _, err = models.InsertGitPackageInfo(&models.GitPackageInfo{ + GitId: goeId, + PackageName: origin.PackageName, + Version: origin.Version, + Summary: origin.Summary, + Decription: origin.Decription, + Status: 0, + }) + + return err } -// Update data to the database -func UpdateYamlToDb(er models.OpenEulerRepoOrigin, branchs map[string]string) { - if len(er.PackageName) > 0 && len(er.Version) > 0 && !strings.Contains(er.Version, "%") { - var oro models.OpenEulerRepoOrigin - oro.PackageName = er.PackageName - oro.Version = er.Version - branch := branchs[er.Version] - repErr := models.QueryEulerRepoOrigin(&oro, "PackageName", "Version") - if oro.Id > 0 { - if len(oro.Decription) < 2 && len(er.Decription) > 2 { - oro.Decription = er.Decription - } - oro.Branchs = branch - if len(oro.Summary) < 2 && len(er.Summary) > 2 { - oro.Summary = er.Summary - } - oro.UpdateTime = common.GetCurTime() - oro.Status = 1 - oro.Repo = er.PackageName - upErr := models.UpdateEulerRepoOrigin(&oro, "Decription", "Branchs", "Summary", "UpdateTime", "Repo") - logs.Info("UpdateEulerRepoOrigin, upErr: ", upErr) - } else { - logs.Info(repErr) - oro.Decription = er.Decription - oro.Branchs = branch - oro.Summary = er.Summary - oro.UpdateTime = common.GetCurTime() - oro.PackageName = er.PackageName - oro.Version = er.Version - oro.Status = 1 - oro.Repo = er.PackageName - oro.CreateTime = common.GetCurTime() - id, inErr := models.InsertEulerRepoOrigin(&oro) - logs.Info("InsertEulerRepoOrigin, id, inErr: ", id, inErr) +// BranchDetail struct of response data +type BranchDetail struct { + Summary string `json:"summary"` + PackageName string `json:"package_name"` + Description interface{} `json:"description"` + Version string `json:"version"` + BRName string `json:"brname"` +} + +func (d BranchDetail) isValid() bool { + outSyncBranch := beego.AppConfig.String("cve::out_of_sync_branch") + outSyncBranchList := strings.Split(outSyncBranch, ",") + for _, v := range outSyncBranchList { + if v == d.BRName { + return false } - } else { - logs.Error("UpdateYamlToDb, The version information of the current warehouse is wrong, er: ", er) } + + if d.PackageName == "webkit2gtk3" { + return false + } + + if d.PackageName != "" && d.Version != "" && !strings.Contains(d.Version, "%") { + return true + } + + return false } -// Synchronize other sources of yaml version -func SyncEulerYaml() { - // Query raw data - yamlId := int64(0) - for { - yamlList := models.GetEulerYamlList(yamlId, 100) - if len(yamlList) > 0 { - for _, yl := range yamlList { - if len(yl.PackageName) < 2 || len(yl.Version) < 1 { - logs.Error("yl: ", yl) - continue - } - yamlId = yl.Id - var ge models.GitOpenEuler - ge.PackageName = yl.PackageName - ge.Version = yl.Version - geErr := models.QueryYamlData(&ge, "PackageName", "Version") - if ge.GitId == 0 { - logs.Info("yaml data does not exist, need to insert data, err: ", - geErr, ",PackageName: ", yl.PackageName, ", Version: ", yl.Version) - AddYamlData(&ge, yl) - } else { - EditYamlData(&ge, yl) - } - } - } else { - break +func (d BranchDetail) generateDescription() string { + switch d.Description.(type) { + case []interface{}: + var t []string + for _, v := range d.Description.([]interface{}) { + t = append(t, v.(string)) } + + return strings.Join(t, ",") + case string: + return d.Description.(string) + default: + return "" } } -// Delete historical yaml source version data that does not exist -func DelHistoryEulerYaml() { - gitId := int64(0) - for { - yamlList := models.GetGitOpenEulerList(gitId, 100) - if len(yamlList) > 0 { - for _, yl := range yamlList { - if len(yl.PackageName) < 2 || len(yl.Version) < 1 { - logs.Error("yl: ", yl) - continue - } - gitId = yl.GitId - var oro models.OpenEulerRepoOrigin - oro.PackageName = yl.PackageName - oro.Version = yl.Version - repErr := models.QueryEulerRepoOrigin(&oro, "PackageName", "Version") - if oro.Id == 0 { - logs.Info("yaml data does not exist, delete data, err: ", - repErr, ",PackageName: ", yl.PackageName, ", Version: ", yl.Version) - yl.Status = 2 - upErr := models.UpdateYamlData(&yl, "Status") - if upErr == nil { - ge := models.GitPackageInfo{PackageName: yl.PackageName, Version: yl.Version} - qErr := models.QueryGitPackageInfo(&ge, "PackageName", "Version") - if qErr == nil { - ge.Status = 2 - upgErr := models.UpdateGitPackageInfo(&ge, "Status") - if upgErr != nil { - logs.Error("UpdateGitPackageInfo, upgErr: ", upgErr) - } - } - } - } +func cleanOldData() error { + if err := models.DeleteEulerRepoOrigin(); err != nil { + return err + } + + if err := models.DeleteGitOpenEuler(); err != nil { + return err + } + + if err := models.DeleteGitPackageInfo(); err != nil { + return err + } + + return nil +} + +func groupByVersion(path string, detail []BranchDetail) map[string]models.OpenEulerRepoOrigin { + split := strings.Split(path, "/") + if len(split) < splitLen { + logs.Error("path error", path) + + return nil + } + + originByVersion := make(map[string]models.OpenEulerRepoOrigin) + for _, d := range detail { + if !d.isValid() { + continue + } + + if ov, ok := originByVersion[d.Version]; ok { + ov.Branchs += fmt.Sprintf(",%s", d.BRName) + + if ov.Summary == "" { + ov.Summary = d.Summary + } + + if ov.Decription == "" { + ov.Decription = d.generateDescription() } + + originByVersion[d.Version] = ov } else { - break + originByVersion[d.Version] = models.OpenEulerRepoOrigin{ + PackageName: d.PackageName, + Version: d.Version, + Summary: d.Summary, + Decription: d.generateDescription(), + Branchs: d.BRName, + Status: 1, + CreateTime: common.GetCurTime(), + UpdateTime: common.GetCurTime(), + Repo: split[1], + } } } -} -func AddYamlData(ge *models.GitOpenEuler, yl models.OpenEulerRepoOrigin) { - ge.PackageName = yl.PackageName - ge.Version = yl.Version - ge.PackageId = 0 - ge.TableId = 0 - //ge.TableName = yl.Branchs - ge.Status = 1 - ymalId, yErr := models.InsertYamlData(ge) - if yErr == nil { - ProcGitPackageInfo(ymalId, yl) - } + return originByVersion } -func EditYamlData(ge *models.GitOpenEuler, yl models.OpenEulerRepoOrigin) { - ge.Status = 1 - yErr := models.UpdateYamlData(ge, "Status") - logs.Info("UpdateYamlData, yErr: ", yErr) - ProcGitPackageInfo(ge.GitId, yl) +// Response struct of response +type Response struct { + Code int `json:"code"` + Message string `json:"message"` + Data []map[string][]BranchDetail `json:"data"` } -func ProcGitPackageInfo(ymalId int64, yl models.OpenEulerRepoOrigin) { - gp := models.GitPackageInfo{GitId: ymalId, PackageName: yl.PackageName, Version: yl.Version} - gpErr := models.QueryGitPackageInfo(&gp, "GitId", "PackageName", "Version") - if gpErr != nil { - gp.GitId = ymalId - gp.PackageName = yl.PackageName - gp.Version = yl.Version - gp.Summary = yl.Summary - gp.Decription = yl.Decription - gp.Status = 0 - id, iErr := models.InsertGitPackageInfo(&gp) - logs.Info("InsertGitPackageInfo, id, iErr", id, iErr) - } else { - gp.Summary = yl.Summary - gp.Decription = yl.Decription - gp.Status = 0 - uErr := models.UpdateGitPackageInfo(&gp, "Summary", "Decription", "Status") - logs.Info("UpdateGitPackageInfo, uErr: ", uErr) +func getEulerPackages() ([]map[string][]BranchDetail, error) { + const retryTimes = 3 + + cli := utils.NewHttpClient(retryTimes) + var result []map[string][]BranchDetail + page := 1 + for { + url := fmt.Sprintf("%s?community=openeuler&pageSize=%d&page=%d", packageUrl, pageSize, page) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + var resp Response + if _, err = cli.ForwardTo(req, &resp); err != nil { + return nil, err + } + + result = append(result, resp.Data...) + + if len(resp.Data) < pageSize { + break + } + + page++ } + + return result, nil }