From 26c84350c9097c7829a215d9e3aa889e9c6f5085 Mon Sep 17 00:00:00 2001 From: "775517776@qq.com" <775517776@qq.com> Date: Wed, 2 Mar 2022 09:32:29 +0800 Subject: [PATCH 1/8] =?UTF-8?q?=E2=80=98add=20time=20task=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cve-vulner-manager/conf/app.conf | 22 +- cve-vulner-manager/controllers/hook.go | 21 +- .../controller/task_controller.go | 121 +++++ .../controller/timer_task_controller.go | 105 ++++ .../cve-timed-task/db_models/db_operations.go | 457 ++++++++++++++++++ .../cve-timed-task/db_models/init_db.go | 13 + .../deletetask/delete_file_task.go | 48 ++ .../downloadfiletask/download_file_task.go | 316 ++++++++++++ .../emailtask/issue_record_email.go | 42 ++ .../cve-timed-task/emailtask/send_email.go | 41 ++ .../cve-timed-task/emailtask/sending_task.go | 40 ++ .../cve-timed-task/gitwebtask/gene_group.go | 137 ++++++ .../cve-timed-task/gitwebtask/yaml_task.go | 6 + cve-vulner-manager/cve-timed-task/start.go | 17 + .../cve-timed-task/tabletask/common.go | 66 +++ .../cve-timed-task/tabletask/crawltask.go | 306 ++++++++++++ .../tabletask/export_excel_task.go | 49 ++ .../cve-timed-task/tabletask/gauss_yaml.go | 290 +++++++++++ .../tabletask/import_excel_task.go | 194 ++++++++ .../cve-timed-task/tabletask/mapping_task.go | 210 ++++++++ .../tabletask/mindspore_yaml.go | 291 +++++++++++ .../tabletask/package_committer_task.go | 133 +++++ .../cve-timed-task/tabletask/repeat_task.go | 95 ++++ .../cve-timed-task/tabletask/run_task.go | 317 ++++++++++++ .../tabletask/spec_error_task.go | 136 ++++++ .../tabletask/supplement_cve.go | 393 +++++++++++++++ .../cve-timed-task/tabletask/to_excel.go | 52 ++ .../cve-timed-task/util/crawl.go | 33 ++ .../cve-timed-task/util/deduplication.go | 10 + .../cve-timed-task/util/makrdir.go | 53 ++ .../cve-timed-task/util/parse_yaml.go | 42 ++ cve-vulner-manager/go.mod | 28 +- cve-vulner-manager/go.sum | 385 ++------------- cve-vulner-manager/main.go | 6 + cve-vulner-manager/models/initdb.go | 9 +- 35 files changed, 4086 insertions(+), 398 deletions(-) create mode 100644 cve-vulner-manager/cve-timed-task/controller/task_controller.go create mode 100644 cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go create mode 100644 cve-vulner-manager/cve-timed-task/db_models/db_operations.go create mode 100644 cve-vulner-manager/cve-timed-task/db_models/init_db.go create mode 100644 cve-vulner-manager/cve-timed-task/deletetask/delete_file_task.go create mode 100644 cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go create mode 100644 cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go create mode 100644 cve-vulner-manager/cve-timed-task/emailtask/send_email.go create mode 100644 cve-vulner-manager/cve-timed-task/emailtask/sending_task.go create mode 100644 cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go create mode 100644 cve-vulner-manager/cve-timed-task/gitwebtask/yaml_task.go create mode 100644 cve-vulner-manager/cve-timed-task/start.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/common.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/crawltask.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/run_task.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go create mode 100644 cve-vulner-manager/cve-timed-task/tabletask/to_excel.go create mode 100644 cve-vulner-manager/cve-timed-task/util/crawl.go create mode 100644 cve-vulner-manager/cve-timed-task/util/deduplication.go create mode 100644 cve-vulner-manager/cve-timed-task/util/makrdir.go create mode 100644 cve-vulner-manager/cve-timed-task/util/parse_yaml.go diff --git a/cve-vulner-manager/conf/app.conf b/cve-vulner-manager/conf/app.conf index 33ed624..06247fc 100644 --- a/cve-vulner-manager/conf/app.conf +++ b/cve-vulner-manager/conf/app.conf @@ -36,17 +36,27 @@ rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEul communityName = "cve-test:1,cve-gauss:2,cve-mindspore:3,open-loo-keng:4" +; [mysql] +; dbhost = "${DB_URI||127.0.0.1}" +; dbport = 3310 +; dbuser = "${DB_USER||cve}" +; dbpwd = "${DB_PWD||***}" +; dbname = oneline_cvevulner +; dbprefix = cve_ +; maxidle = 30 +; maxconn = 3000 + + [mysql] -dbhost = "${DB_URI||127.0.0.1}" -dbport = 3310 -dbuser = "${DB_USER||cve}" -dbpwd = "${DB_PWD||***}" -dbname = oneline_cvevulner +dbhost = "127.0.0.1" +dbport = 3306 +dbuser = "root" +dbpwd = "root" +dbname = cvevulner dbprefix = cve_ maxidle = 30 maxconn = 3000 - [log] log_level = 7 log_dir = ./logs diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index 079729e..01e1a91 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -3,6 +3,7 @@ package controllers import ( "bytes" "cvevulner/common" + "cvevulner/cve-timed-task/tabletask" "cvevulner/models" "cvevulner/task" "cvevulner/taskhandler" @@ -1783,25 +1784,11 @@ func PostCveAgency(issueTmp models.IssueTemplate, agencyPram AgencyPrams) { func PostTriggerGetCve(issueTmp models.IssueTemplate, owner, accessToken, cuAccount string) { // Trigger third-party data generation - pullCveUrl := beego.AppConfig.String("reflink::get_cve_link") - pullCveReq := make(map[string]interface{}, 0) cc := "" if len(issueTmp.CveNum) > 0 { - pullCveReq["CveNum"] = issueTmp.CveNum - respData, err := util.HTTPPostCom(pullCveReq, pullCveUrl) - if err == nil { - var rd map[string]interface{} - err = json.Unmarshal(respData, &rd) - if err != nil { - logs.Error("err: ", err, string(respData)) - cc = fmt.Sprintf(CommentGetNvdCveFailed, cuAccount) - } else { - if int(rd["code"].(float64)) == 200 { - cc = fmt.Sprintf(CommentGetNvdCveSuccess, cuAccount) - } else { - cc = fmt.Sprintf(CommentGetNvdCveFailed, cuAccount) - } - } + resp := tabletask.PullCve(issueTmp.CveNum) + if resp == 1 { + cc = fmt.Sprintf(CommentGetNvdCveSuccess, cuAccount) } else { cc = fmt.Sprintf(CommentGetNvdCveFailed, cuAccount) } diff --git a/cve-vulner-manager/cve-timed-task/controller/task_controller.go b/cve-vulner-manager/cve-timed-task/controller/task_controller.go new file mode 100644 index 0000000..43e56ee --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/controller/task_controller.go @@ -0,0 +1,121 @@ +package controller + +import ( + "cvevulner/cve-timed-task/deletetask" + "cvevulner/cve-timed-task/emailtask" + "cvevulner/cve-timed-task/gitwebtask" + "cvevulner/cve-timed-task/tabletask" + "cvevulner/cve-timed-task/util" + "github.com/astaxie/beego/logs" + "time" +) + +// GwController Grab warehouse yaml data +func GwController() { + logs.Info("Start of crawling organization, warehouse, owners correspondence relationship timing task") + gitwebtask.Handle() + //TODO + logs.Info("Grab warehouse yaml data task completed") +} + +// RunTableTask Parse the artificial CVE table, crawl the CVE official +//website data, and store it in the database +func RunTableTask() { + logs.Info("Analyze the manual data table, crawl the CVE official website data, and start the timing task") + tabletask.AddErrorDetails() + tabletask.HandleDate() + tabletask.RepeatTask() + tabletask.GetPublishedDateTask() + logs.Info("Analyze the manual data table, crawl the CVE official website data task completed") +} + +// ToExcelController Query the data whose cve_status status is 3 or 4 in the +// database to generate excel Modify the status of cve_status to 3 or 4 to 5, +// indicating that it has been added to excel to be processed +func ToExcelController() { + logs.Info("Export data to generate excel, send error data feedback email task start") + filePath := "problemexcels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return + } + createTime := time.Now().Format("2006-01-02") + + tabletask.GenerateTable(filePath + "error_data_" + createTime + ".xls") + emailtask.SendingTypeTwo(filePath) + logs.Info("To excel and send email task completed") +} + +// RunMappingTask Parse the artificial CVE table, crawl the CVE +//official website data, and store it in the database +func RunMappingTask() { + logs.Info("The timing task of parsing the package name mapping table starts") + tabletask.ToMysql() + logs.Info("The timing task of parsing the package name mapping table completed") +} + +// RunDelFileTask Delete files older than one month +func RunDelFileTask() { + logs.Info("Clean up the folder timed task starts") + deletetask.DelDir("oldexcels/") + logs.Info("Clean up the folder task completed") +} + +// RunIssueRecordTask issues record email task +func RunIssueRecordTask() { + logs.Info("issues record email task start") + filePath := "export_excels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.MakeDir error:", err, filePath) + return + } + tabletask.GenerateExcels("1", filePath+"create_issues.xls") + tabletask.GenerateExcels("2", filePath+"unctreated_issues.xls") + emailtask.Sending(filePath) + logs.Info("issues record email task complete") +} + +// RunWhitelistTask the task of parsing package whitelist +func RunWhitelistTask() { + logs.Info("the task of parsing package whitelist start") + tabletask.ImportDataTypeTwo() + logs.Info("the task of parsing package whitelist complete") +} + +// RunPackageCommitterTask the task of parsing package_committer excels +func RunPackageCommitterTask() { + logs.Info("the task of parsing package_committer_excels start") + tabletask.ImportData() + logs.Info("the task of parsing package_committer_excels complete") +} + +// SupplementCveTask Complete the template information of the issue with the data on the CVE official website +func SupplementCveTask() { + logs.Info("Supplemental cve information task starts") + tabletask.SupplementCve() + logs.Info("Supplement cve information task is over") +} + +// LongSupplementCveTask Complete the template information of the +//issue with the data on the CVE official website +func LongSupplementCveTask() { + logs.Info("long Supplemental cve information task starts") + tabletask.LongSupplementCve() + logs.Info("long Supplement cve information task is over") +} + +// ParseOpenGaussYamlTask Download the yaml file first, then parse the yaml file of opengauss +func ParseOpenGaussYamlTask() { + logs.Info("the task of Process openGauss yaml data start") + tabletask.ProcGaussYaml() + logs.Info("the task of Process openGauss yaml data complete") +} + +// ParseMindsporeYamlTask Download the yaml file first, then parse the yaml file of mindspore +func ParseMindsporeYamlTask() { + logs.Info("the task of Process mindspore yaml data start") + tabletask.ProcMindSporeYaml() + logs.Info("the task of Process mindspore yaml data complete") +} diff --git a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go new file mode 100644 index 0000000..836aca4 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go @@ -0,0 +1,105 @@ +package controller + +import ( + "github.com/astaxie/beego/logs" + "github.com/robfig/cron/v3" +) + +func StartTimerTask() error { + c := cron.New() + + //Warehouse, organization, and owner correspondence timing task + IdGwController, err := c.AddFunc("2 0 * * 0-6", GwController) + if err != nil { + logs.Error("add GwController task error:", err.Error()) + return err + } + logs.Info("Add ToExcelController task success, ID is:", IdGwController) + + IdRunTableTask, err := c.AddFunc("@every 40m", RunTableTask) + if err != nil { + logs.Error("add RunTableTask task error:", err.Error()) + return err + } + logs.Info("Add RunTableTask task success, ID is:", IdRunTableTask) + + //Error data export and email sending timing task + IdToExcelController, err := c.AddFunc("2 0 * * 0-6", ToExcelController) + if err != nil { + logs.Error("add ToExcelController task error:", err.Error()) + return err + } + logs.Info("Add ToExcelController task success, ID is:", IdToExcelController) + + //Package name correspondence relationship timing task + IdRunMappingTask, err := c.AddFunc("30 7 * * 0-6", RunMappingTask) + if err != nil { + logs.Error("add RunMappingTask task error:", err.Error()) + return err + } + logs.Info("Add RunMappingTask task success, ID is:", IdRunMappingTask) + + //Delete ,iles timed tasks that are more than one month old + IdRunDelFileTask, err := c.AddFunc("30 9 * * 0-6", RunDelFileTask) + if err != nil { + logs.Error("add RunDelFileTask task error:", err.Error()) + return err + } + logs.Info("Add RunDelFileTask task success, ID is:", IdRunDelFileTask) + + //issue record email feedback + IdRunIssueRecordTask, err := c.AddFunc("30 7 * * 0-6", RunIssueRecordTask) + if err != nil { + logs.Error("add RunIssueRecordTask task error:", err.Error()) + return err + } + logs.Info("Add RunIssueRecordTask task success, ID is:", IdRunIssueRecordTask) + + //analysis the table package whitelist + IdRunWhitelistTask, err := c.AddFunc("@every 2h", RunWhitelistTask) + if err != nil { + logs.Error("add RunWhitelistTask task error:", err.Error()) + return err + } + logs.Info("Add RunWhitelistTask task success, ID is:", IdRunWhitelistTask) + + //parsing the table package_committer_excels + IdRunPackageCommitterTask, err := c.AddFunc("@every 1h", RunPackageCommitterTask) + if err != nil { + logs.Error("add RunPackageCommitterTask task error:", err.Error()) + return err + } + logs.Info("Add RunPackageCommitterTask task success, ID is:", IdRunPackageCommitterTask) + + //Complete the template information of the issue with the data on the CVE official website + IdSupplementCveTask, err := c.AddFunc("@every 27m", SupplementCveTask) + if err != nil { + logs.Error("add SupplementCveTask task error:", err.Error()) + return err + } + logs.Info("Add IdParseOpenGaussYamlTask task success, ID is:", IdSupplementCveTask) + + IdLongSupplementCveTask, err := c.AddFunc("30 1 * * 0-6", LongSupplementCveTask) + if err != nil { + logs.Error("add LongSupplementCveTask task error:", err.Error()) + return err + } + logs.Info("Add IdLongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) + //Parse opengauss yaml file + IdParseOpenGaussYamlTask, err := c.AddFunc("30 2 * * 0-6", ParseOpenGaussYamlTask) + if err != nil { + logs.Error("add ParseOpenGaussYamlTask task error:", err.Error()) + return err + } + logs.Info("Add IdParseOpenGaussYamlTask task success, ID is:", IdParseOpenGaussYamlTask) + + //Parse the yaml file of mindspore + IdParseMindsporeYamlTask, err := c.AddFunc("30 3 * * 0-6", ParseMindsporeYamlTask) + if err != nil { + logs.Error("add ParseMindsporeYamlTask task error:", err.Error()) + return err + } + logs.Info("Add IdParseOpenGaussYamlTask task success, ID is:", IdParseMindsporeYamlTask) + c.Start() + return nil +} diff --git a/cve-vulner-manager/cve-timed-task/db_models/db_operations.go b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go new file mode 100644 index 0000000..065dd56 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go @@ -0,0 +1,457 @@ +package db_models + +import ( + "cvevulner/models" + "github.com/astaxie/beego/orm" + "time" +) + +// UpdateYamlOriginMark Update origin data +func UpdateYamlOriginMark() error { + sqlString := "update cve_open_guss_yaml set mark_bit = 2 where mark_bit = 1" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +// SelectYamlOriginData Query origin data +func SelectYamlOriginData(packageName, version string) (models.OpenGussYaml, error) { + var openGussYaml models.OpenGussYaml + sqlString := "SELECT id FROM cve_open_guss_yaml WHERE package_name = ? and version = ?" + err := ORM.Raw(sqlString, packageName, version).QueryRow(&openGussYaml) + return openGussYaml, err +} + +// InsertYamlOriginData insert origin data +func InsertYamlOriginData(openGussYaml *models.OpenGussYaml) error { + sqlString := "insert into cve_open_guss_yaml(package_name,version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) values(?,?,?,?,?,?,?,?,?)" + _, err := ORM.Raw(sqlString, openGussYaml.PackageName, openGussYaml.Version, openGussYaml.OriginUrl, openGussYaml.Status, openGussYaml.CpeName, openGussYaml.CreateTime, openGussYaml.UpdateTime, openGussYaml.MarkBit, openGussYaml.Repo).Exec() + return err +} + +// UpdateYamlOriginData Update origin data +func UpdateYamlOriginData(openGussYaml *models.OpenGussYaml) (int64, error) { + id, err := ORM.Update(openGussYaml, "OriginUrl", "Status", "CpeName", "UpdateTime", "Repo", "MarkBit") + return id, err +} + +// SelectYamlData Query data +func SelectYamlData(packageName, version string) ([]models.GitOpenEuler, int64, error) { + var gitOpenEulerList []models.GitOpenEuler + sqlString := "SELECT package_id, git_id FROM cve_git_open_euler WHERE package_name = ? and version = ? order by package_id desc" + rowsAffected, err := ORM.Raw(sqlString, packageName, version).QueryRows(&gitOpenEulerList) + return gitOpenEulerList, rowsAffected, err +} + +// SelectYamlLastData Query the last data +func SelectYamlLastData() (models.GitOpenEuler, error) { + var gitOpenEuler models.GitOpenEuler + sqlString := "SELECT package_id FROM cve_git_open_euler order by git_id desc" + err := ORM.Raw(sqlString).QueryRow(&gitOpenEuler) + return gitOpenEuler, err +} + +// InsertYamlData insert data +func InsertYamlData(gitOpenEuler *models.GitOpenEuler) (int64, error) { + sqlString := "insert into cve_git_open_euler(package_id,package_name,version,origin_url,create_time,update_time,cpe_packname,status) values(?,?,?,?,?,?,?,?)" + result, err := ORM.Raw(sqlString, gitOpenEuler.PackageId, gitOpenEuler.PackageName, gitOpenEuler.Version, gitOpenEuler.OriginUrl, gitOpenEuler.CreateTime, gitOpenEuler.UpdateTime, gitOpenEuler.CpePackName, gitOpenEuler.Status).Exec() + id, _ := result.LastInsertId() + return id, err +} + +// InsertYamlDetailData insert data details +func InsertYamlDetailData(gitPackageInfo *models.GitPackageInfo) error { + sqlString := "insert into cve_git_package_info(git_id,package_name,version,origin_url,create_time,update_time,decription,status) values(?,?,?,?,?,?,?,?)" + _, err := ORM.Raw(sqlString, gitPackageInfo.GitId, gitPackageInfo.PackageName, gitPackageInfo.Version, gitPackageInfo.OriginUrl, gitPackageInfo.CreateTime, gitPackageInfo.UpdateTime, gitPackageInfo.Decription, gitPackageInfo.Status).Exec() + return err +} + +// DeleteYamlOpenEulerDetailData delete origin data +func DeleteYamlOpenEulerDetailData(gitId int64) error { + sqlString := "delete from cve_git_package_info where git_id = ?" + _, err := ORM.Raw(sqlString, gitId).Exec() + return err +} + +// DeleteYamlOpenEulerData delete origin data +func DeleteYamlOpenEulerData(gitId int64) error { + sqlString := "delete from cve_git_open_euler where git_id = ?" + _, err := ORM.Raw(sqlString, gitId).Exec() + return err +} + +// UpdateYamlData update data +func UpdateYamlData(gitOpenEuler *models.GitOpenEuler) error { + sqlString := "update cve_git_open_euler set origin_url = ?,update_time = ?,cpe_packname = ?,status=? where package_id = ? and package_name = ? and version = ?" + _, err := ORM.Raw(sqlString, gitOpenEuler.OriginUrl, gitOpenEuler.UpdateTime, gitOpenEuler.CpePackName, gitOpenEuler.Status, gitOpenEuler.PackageId, gitOpenEuler.PackageName, gitOpenEuler.Version).Exec() + return err +} + +// UpdateYamlDetailData Update data details +func UpdateYamlDetailData(gitPackageInfo *models.GitPackageInfo) error { + sqlString := "update cve_git_package_info set package_name = ?,version = ?,origin_url = ?, update_time = ?,status=? where git_id = ?" + _, err := ORM.Raw(sqlString, gitPackageInfo.PackageName, gitPackageInfo.Version, gitPackageInfo.OriginUrl, gitPackageInfo.UpdateTime, gitPackageInfo.Status, gitPackageInfo.GitId).Exec() + return err +} + +// DeleteYamlOriginMark delete origin data +func DeleteYamlOriginMark() error { + sqlString := "delete from cve_open_guss_yaml where mark_bit = 2" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +// SelectOpenEulerYamlData Query data +func SelectOpenEulerYamlData() ([]models.GitOpenEuler, int64, error) { + var gitOpenEulerList []models.GitOpenEuler + sqlString := "SELECT package_id, git_id,package_name,version FROM cve_git_open_euler WHERE package_id >= ?" + rowsAffected, err := ORM.Raw(sqlString, 10000000).QueryRows(&gitOpenEulerList) + return gitOpenEulerList, rowsAffected, err +} + +// SelectGaussYamlOriginData Query origin data +func SelectGaussYamlOriginData(packageName, version string) (models.OpenGussYaml, error) { + var openGussYaml models.OpenGussYaml + sqlString := "SELECT id FROM cve_open_guss_yaml WHERE package_name = ? and version = ?" + err := ORM.Raw(sqlString, packageName, version).QueryRow(&openGussYaml) + return openGussYaml, err +} + +// SelectMindSporeYamlOriginData Query origin data +func SelectMindSporeYamlOriginData(packageName, version string) (models.MindSporeYaml, error) { + var mindSporeYaml models.MindSporeYaml + sqlString := "SELECT id FROM cve_mind_spore_yaml WHERE package_name = ? and version = ?" + err := ORM.Raw(sqlString, packageName, version).QueryRow(&mindSporeYaml) + return mindSporeYaml, err +} + +// SelectOpenLookengYamlOriginData Query origin data +func SelectOpenLookengYamlOriginData(packageName, version string) (models.OpenLookengYaml, error) { + var openLookengYaml models.OpenLookengYaml + sqlString := "SELECT id FROM cve_open_lookeng_yaml WHERE package_name = ? and version = ?" + err := ORM.Raw(sqlString, packageName, version).QueryRow(&openLookengYaml) + return openLookengYaml, err +} + +func GetOneGitRepoGroups(sigName string) (models.GitRepoGroups, error) { + var gitRepsGroups models.GitRepoGroups + sqlString := "select * from cve_git_repo_groups where group_name = ?" + err := ORM.Raw(sqlString, sigName).QueryRow(&gitRepsGroups) + return gitRepsGroups, err +} + +func InsertOneGitRepoGroups(gitRepoGroups *models.GitRepoGroups) error { + sqlString := "insert into cve_git_repo_groups (group_name) values (?)" + _, err := ORM.Raw(sqlString, gitRepoGroups.GroupName).Exec() + return err +} + +func DeleteGitRepoMember(groupId int64) error { + sqlString := "delete from cve_gite_repo_member where group_id = ?" + _, err := ORM.Raw(sqlString, groupId).Exec() + return err +} + +func InsertOneGiteRepoMember(giteRepoMember *models.GiteRepoMember) error { + sqlString := "insert into cve_gite_repo_member (group_id,member_name,member_type,create_time) values (?, ?, ?, ?)" + _, err := ORM.Raw(sqlString, giteRepoMember.GroupId, giteRepoMember.MemberName, giteRepoMember.MemberType, giteRepoMember.CreateTime).Exec() + return err +} + +func GetCveScoreListByBeforeDate(beforeDate string) ([]models.Score, int64, error) { + var scoreList []models.Score + sqlString := "select cve_id,cve_num from cve_score where nvd_score = 0 and update_time >= ? order by update_time desc" + rowsAffected, err := ORM.Raw(sqlString, beforeDate).QueryRows(&scoreList) + return scoreList, rowsAffected, err +} + +func GetCveVulnCenterList(cveDesc, repairTime, updateTime string) ([]models.VulnCenter, int64, error) { + var vulnCenterList []models.VulnCenter + sqlString := "select cve_id,cve_num from cve_vuln_center where (cve_desc = ? or repair_time = ?) and update_time >= ? order by cve_id desc" + rowsAffected, err := ORM.Raw(sqlString, cveDesc, repairTime, updateTime).QueryRows(&vulnCenterList) + return vulnCenterList, rowsAffected, err +} + +func GetCveIssueTemplate(nvdScore, nvdVector, cveBrief, updateTime string) ([]models.IssueTemplate, int64, error) { + var issueTemplateList []models.IssueTemplate + sqlString := "select cve_id,cve_num from cve_issue_template where (nvd_score = ? or nvd_vector = ? or cve_brief = ?) and update_time >= ? order by cve_id desc" + rowsAffected, err := ORM.Raw(sqlString, nvdScore, nvdVector, cveBrief, updateTime).QueryRows(&issueTemplateList) + return issueTemplateList, rowsAffected, err +} + +func GetOneVulnCenter(cveId, cveNum string) (models.VulnCenter, error) { + var VulnCenter models.VulnCenter + sqlString := "select cve_num, pack_name, cve_version, cve_desc,repair_time,cve_status,cve_id from cve_vuln_center where cve_id = ? and cve_num = ? order by cve_id desc" + err := ORM.Raw(sqlString, cveId, cveNum).QueryRow(&VulnCenter) + return VulnCenter, err +} + +func UpdateVulnCenterTypeOne(vulnCenter models.VulnCenter) error { + sqlString := "update cve_vuln_center set repair_time = ?,update_time=?, cve_status=? where cve_id=?" + _, err := ORM.Raw(sqlString, vulnCenter.RepairTime, vulnCenter.UpdateTime, vulnCenter.Status, vulnCenter.CveId).Exec() + return err +} + +func UpdateVulnCenterTypeTwo(vulnCenter models.VulnCenter) error { + sqlString := "update cve_vuln_center set cve_desc = ?,update_time=?, cve_status=? where cve_id=?" + _, err := ORM.Raw(sqlString, vulnCenter.Description, vulnCenter.UpdateTime, vulnCenter.Status, vulnCenter.CveId).Exec() + return err +} + +func GetCveScoreByCveIdAndCveNum(cveId int64, cveNum string) (models.Score, error) { + var score models.Score + sqlString := "select id,nvd_score from cve_score where cve_id = ? and cve_num = ?" + err := ORM.Raw(sqlString, cveId, cveNum).QueryRow(&score) + return score, err +} + +func UpdateCveScore(score models.Score) error { + sqlString := "update cve_score set nvd_score=?, n_vector_value=?, n_attack_vector=?, n_access_vector=?, n_attack_complexity=?, n_access_complexity=?, n_privilege_required=?, n_user_interaction=?, n_scope=?, n_confidentiality=?, n_integrity=?, n_availability=?, n_authentication=?, update_time=?,score_type=? where id = ?" + _, err := ORM.Raw(sqlString, score.NVDScore, score.NvectorVule, score.NattackVector, score.NaccessVector, score.NattackComplexity, score.NaccessComplexity, score.NprivilegeRequired, score.NuserInteraction, score.Nscope, score.Nconfidentiality, score.Nintegrity, score.Navailability, score.Nauthentication, score.UpdateTime, score.ScoreType, score.Id).Exec() + return err +} + +func GetIssueTemplateTypeOne(issueTemplate models.IssueTemplate) (models.IssueTemplate, error) { + var issueTemplateResp models.IssueTemplate + sqlString := "select template_id,nvd_score,nvd_vector,cve_brief from cve_issue_template where (nvd_score = ? or nvd_vector = ? or cve_brief = ?) and cve_id = ? and cve_num = ?" + err := ORM.Raw(sqlString, issueTemplate.NVDScore, issueTemplate.NVDVector, issueTemplate.CveBrief, issueTemplate.CveId, issueTemplate.CveNum).QueryRow(&issueTemplateResp) + return issueTemplateResp, err +} + +func UpdateCveIssueTemplateTypeOne(issueTemplate models.IssueTemplate) error { + sqlString := "update cve_issue_template set nvd_score = ?, update_time=? where template_id = ?" + _, err := ORM.Raw(sqlString, issueTemplate.NVDScore, issueTemplate.UpdateTime, issueTemplate.TemplateId).Exec() + return err +} + +func UpdateCveIssueTemplateTypeTwo(issueTemplate models.IssueTemplate) error { + sqlString := "update cve_issue_template set nvd_vector = ?, update_time=? where template_id = ? " + _, err := ORM.Raw(sqlString, issueTemplate.NVDVector, issueTemplate.UpdateTime, issueTemplate.TemplateId).Exec() + return err +} + +func UpdateCveIssueTemplateTypeThree(issueTemplate models.IssueTemplate) error { + sqlString := "update cve_issue_template set cve_brief = ?, update_time=? where template_id = ? " + _, err := ORM.Raw(sqlString, issueTemplate.CveBrief, issueTemplate.UpdateTime, issueTemplate.TemplateId).Exec() + return err +} + +func UpdateVulnCenter(vulnCenter models.VulnCenter) error { + sqlString := "update cve_vuln_center set update_time=?, cve_status=? where cve_id=?" + _, err := ORM.Raw(sqlString, vulnCenter.UpdateTime, vulnCenter.Status, vulnCenter.CveId).Exec() + return err +} + +func GetSpceError(cveNum, cveOwner, packName string) (models.SpecError, error) { + var spceError models.SpecError + sqlString := "select * from cve_spec_error where cve_num = ? and cve_owner = ? and pack_name = ?" + err := ORM.Raw(sqlString, cveNum, cveOwner, packName).QueryRow(&spceError) + return spceError, err +} + +func UpdateCveOriginExcel(originExcel models.OriginExcel) error { + sqlString := "update cve_origin_excel set cve_desc = ?, cve_status = ? where cve_num= ? and pack_name = ? and cve_version = ?" + _, err := ORM.Raw(sqlString, originExcel.CveDesc, originExcel.CveStatus, originExcel.CveNum, originExcel.PackName, originExcel.CveVersion).Exec() + return err +} + +// UpdateMindYamlOriginMark UpdateYamlOriginMarkLookeng Update origin data +func UpdateMindYamlOriginMark() error { + sqlString := "update cve_open_lookeng_yaml set mark_bit = 2 where mark_bit = 1" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +// SelectMindYamlOriginData Query origin data +func SelectMindYamlOriginData(packageName, version, repokey string) (models.OpenGussYaml, error) { + var openGussYaml models.OpenGussYaml + sqlString := "SELECT id FROM cve_mind_spore_yaml WHERE package_name = ? and version = ? and repo_name = ?" + err := ORM.Raw(sqlString, packageName, version, repokey).QueryRow(&openGussYaml) + return openGussYaml, err +} + +// InsertMindYamlOriginData insert origin data +func InsertMindYamlOriginData(mindSporeYaml *models.MindSporeYaml) error { + sqlString := "insert into cve_mind_spore_yaml(package_name,version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) values(?,?,?,?,?,?,?,?,?)" + _, err := ORM.Raw(sqlString, mindSporeYaml.PackageName, mindSporeYaml.Version, mindSporeYaml.OriginUrl, mindSporeYaml.Status, mindSporeYaml.CpeName, mindSporeYaml.CreateTime, mindSporeYaml.UpdateTime, mindSporeYaml.MarkBit, mindSporeYaml.Repo).Exec() + return err +} + +// UpdateMindYamlOriginData Update origin data +func UpdateMindYamlOriginData(mindSporeYaml *models.MindSporeYaml) (int64, error) { + id, err := ORM.Update(mindSporeYaml, "OriginUrl", "Status", "CpeName", "UpdateTime", "MarkBit") + return id, err +} + +// DeleteMindYamlOriginMark delete origin data +func DeleteMindYamlOriginMark() error { + sqlString := "delete from cve_mind_spore_yaml where mark_bit = 2" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +func SelectCveFileHashByFileName(fileName string) ([]interface{}, int64, error) { + var list orm.ParamsList + sqlString := "select file_hash from cve_file_hash where file_name = ?" + num, err := ORM.Raw(sqlString, fileName).ValuesFlat(&list) + return list, num, err +} + +func InsertCveFileHash(fileName, fileHash string) error { + sqlString := "insert into cve_file_hash (file_name, file_hash) values (?, ?)" + _, err := ORM.Raw(sqlString, fileName, fileHash).Exec() + return err +} + +func InsertSpecIssueAssigness(specIssueAssigness *models.SpecIssueAssigness) error { + sqlString := "insert ignore into cve_spec_issue_assigness (package_name, issue_assignee, status, create_time) values (?, ?, ?, ?)" + _, err := ORM.Raw(sqlString, specIssueAssigness.PackageName, specIssueAssigness.Assignee, specIssueAssigness.Status, specIssueAssigness.CreateTime).Exec() + return err +} + +func SelectIssueRepoWhiteListByPackageName(packageName string) (models.IssueRepoWhitelist, error) { + var issueRepoWhiteList models.IssueRepoWhitelist + sqlString := "select * from cve_issue_repo_whitelist where package_name = ?" + err := ORM.Raw(sqlString, packageName).QueryRow(&issueRepoWhiteList) + return issueRepoWhiteList, err +} + +func InsertIssueRepoWhiteList(issueRepoWhiteList *models.IssueRepoWhitelist) error { + sqlString := "insert into cve_issue_repo_whitelist (package_name, version, status, branchs, create_time, update_time, delete_time) values (?, ?, ?, ?, ?, ?, ?)" + _, err := ORM.Raw(sqlString, issueRepoWhiteList.PackageName, issueRepoWhiteList.Version, issueRepoWhiteList.Status, issueRepoWhiteList.Branchs, issueRepoWhiteList.CreateTime, issueRepoWhiteList.UpdateTime, issueRepoWhiteList.DeleteTime).Exec() + return err +} + +func UpdateIssueRepoWhiteList(issueRepoWhiteList models.IssueRepoWhitelist) error { + sqlString := "update cve_issue_repo_whitelist set status = ?, branchs = ?, update_time = ? where package_name = ? and version = ?" + _, err := ORM.Raw(sqlString, issueRepoWhiteList.Status, issueRepoWhiteList.Branchs, issueRepoWhiteList.UpdateTime, issueRepoWhiteList.PackageName, issueRepoWhiteList.Version).Exec() + return err +} + +func GetResults(status string) ([]models.IssueCreateRecord, int64, error) { + lastMonthDate := time.Now().AddDate(0, -1, 0).Format("2006-01-02") + var issueCreateRecordList []models.IssueCreateRecord + sqlString := "select * from cve_issue_create_record where status = ? and create_time > ?" + rowsAffected, err := ORM.Raw(sqlString, status, lastMonthDate).QueryRows(&issueCreateRecordList) + return issueCreateRecordList, rowsAffected, err +} + +func UpdateStatue(status string) error { + sqlString := "update cve_issue_create_record set status = ?,update_time = ? where status = ?" + _, err := ORM.Raw(sqlString, 4, time.Now().Format("2006-01-02 15:04:05"), status).Exec() + return err +} + +func GetCveEmailListByEmailType(emailType string) ([]models.EmailList, int64, error) { + var emailList []models.EmailList + sqlString := "select email_name from cve_email_list where email_type=?" + rowsAffected, err := ORM.Raw(sqlString, emailType).QueryRows(&emailList) + return emailList, rowsAffected, err +} + +func SelectCvePackAgeCpeByPackName(packName string) (models.PackageCpe, error) { + var packageCpe models.PackageCpe + sqlString := "select * from cve_package_cpe where packname = ?" + err := ORM.Raw(sqlString, packName).QueryRow(&packageCpe) + return packageCpe, err +} + +func InsertCvePackAgeCpe(packageCpe *models.PackageCpe) error { + sqlString := "insert into cve_package_cpe (packname,cpe_packname,create_time) values (?,?,?)" + _, err := ORM.Raw(sqlString, packageCpe.PackName, packageCpe.CpePackName, packageCpe.CreateTime).Exec() + return err +} + +func UpdateCvePackAgeCpe(packageCpe models.PackageCpe) error { + sqlString := "update cve_package_cpe set cpe_packname=?, create_time=? where packname=?" + _, err := ORM.Raw(sqlString, packageCpe.CpePackName, packageCpe.CreateTime, packageCpe.PackName).Exec() + return err +} + +func GetCveOriginExcel() ([]models.OriginExcel, int64, error) { + lastMonthDate := time.Now().AddDate(0, -1, 0).Format("2006-01-02") + var originExcel []models.OriginExcel + sqlString := "select * from cve_origin_excel where (cve_status = 3 or cve_status=4) and is_export = 1 and create_time > ?" + rowsAffected, err := ORM.Raw(sqlString, lastMonthDate).QueryRows(&originExcel) + return originExcel, rowsAffected, err +} + +func UpdateStatusTypeTwo() error { + sqlString := "update cve_origin_excel set is_export=2 where cve_status=3 or cve_status=4" + _, err := ORM.Raw(sqlString).Exec() + return err +} + +func GetCveSpecError(cveNum string) (models.SpecError, error) { + var specError models.SpecError + sqlString := "SELECT * FROM cve_spec_error where cve_num = ?" + err := ORM.Raw(sqlString, cveNum).QueryRow(&specError) + return specError, err +} + +func InsertCveSpecError(specError *models.SpecError) (int64, error) { + return ORM.Insert(specError) +} + +func UpdateCveSpecError(specError models.SpecError) error { + sqlString := "update cve_spec_error set cve_desc = ?, cve_owner = ?, cve_status = ?, pack_name = ?, update_time = ? where cve_num = ?" + _, err := ORM.Raw(sqlString, specError.Description, specError.Owner, specError.Status, specError.PackName, specError.UpdateTime, specError.CveNum).Exec() + return err +} + +func GetCveOriginExcelTypeTwo(cveNum, packName, cveVersion string) (models.OriginExcel, error) { + var originExcel models.OriginExcel + sqlString := "select * from cve_origin_excel where cve_num= ? and pack_name = ? and cve_version = ?" + err := ORM.Raw(sqlString, cveNum, packName, cveVersion).QueryRow(&originExcel) + return originExcel, err +} + +func UpdateCveOriginExcelTypeThree(originExcel models.OriginExcel) error { + sqlString := "update cve_origin_excel set nvd_score=?, cve_level=?, cve_desc=?, repair_time=?, vector_value=?, attack_vector=?, access_vector=?, attack_complexity=?, access_complexity=?, privilege_required=?, user_interaction=?, scope=?, confidentiality=?, integrity=?, availability=?, authentication=?, cve_status=?, update_time=? where cve_num=? and pack_name = ? and cve_version = ?" + _, err := ORM.Raw(sqlString, originExcel.NVDScore, originExcel.CveLevel, originExcel.CveDesc, originExcel.RepairTime, originExcel.NVDVector, originExcel.AttackVector, originExcel.AccessVector, originExcel.AttackComplexity, originExcel.AccessComplexity, originExcel.PrivilegeRequired, originExcel.UserInteraction, originExcel.Scope, originExcel.Confidentiality, originExcel.Integrity, originExcel.Availability, originExcel.Authentication, originExcel.CveStatus, originExcel.UpdateTime, originExcel.CveNum, originExcel.PackName, originExcel.CveVersion).Exec() + return err +} + +func UpdateCveOriginExcelTypeFour(originExcel models.OriginExcel) error { + sqlString := "update cve_origin_excel set nvd_score=?, cve_level=?, cve_desc=?, repair_time=?, vector_value=?, attack_vector=?, access_vector=?, attack_complexity=?, access_complexity=?, privilege_required=?, user_interaction=?, scope=?, confidentiality=?, integrity=?, availability=?, authentication=?, cve_status=?, update_time=?, score_type=? where cve_num=? and pack_name = ? and cve_version = ?" + _, err := ORM.Raw(sqlString, originExcel.NVDScore, originExcel.CveLevel, originExcel.CveDesc, originExcel.RepairTime, originExcel.NVDVector, originExcel.AttackVector, originExcel.AccessVector, originExcel.AttackComplexity, originExcel.AccessComplexity, originExcel.PrivilegeRequired, originExcel.UserInteraction, originExcel.Scope, originExcel.Confidentiality, originExcel.Integrity, originExcel.Availability, originExcel.Authentication, originExcel.CveStatus, originExcel.UpdateTime, originExcel.ScoreType, originExcel.CveNum, originExcel.PackName, originExcel.CveVersion).Exec() + return err +} + +func UpdateCveOriginExcelTypeFive(originExcel models.OriginExcel) error { + sqlString := "update cve_origin_excel set cve_desc = ?, cve_status = ? where cve_num = ?" + _, err := ORM.Raw(sqlString, originExcel.CveDesc, originExcel.CveStatus, originExcel.CveNum).Exec() + return err +} + +func InsertCveOriginExcel(originExcel *models.OriginExcel) error { + sqlString := "INSERT INTO cve_origin_excel (cve_num, cve_url, cve_version, pack_name, score_type, nvd_score, cve_level, cve_desc, repair_time, vector_value, attack_vector, access_vector, attack_complexity, access_complexity, privilege_required, user_interaction, scope, confidentiality, integrity, availability, authentication, cve_status, create_time, update_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + _, err := ORM.Raw(sqlString, originExcel.CveNum, originExcel.CveUrl, originExcel.CveVersion, originExcel.PackName, originExcel.ScoreType, originExcel.NVDScore, originExcel.CveLevel, originExcel.CveDesc, originExcel.RepairTime, originExcel.NVDVector, originExcel.AttackVector, originExcel.AccessVector, originExcel.AttackComplexity, originExcel.AccessComplexity, originExcel.PrivilegeRequired, originExcel.UserInteraction, originExcel.Scope, originExcel.Confidentiality, originExcel.Integrity, originExcel.Availability, originExcel.Authentication, originExcel.CveStatus, originExcel.CreateTime, originExcel.UpdateTime).Exec() + return err +} + +func SelectCveOriginExcel(scoreType string) ([]models.OriginExcel, int64, error) { + var originExcel []models.OriginExcel + sqlString := "SELECT cve_num,pack_name,cve_version FROM cve_origin_excel WHERE score_type = ? OR score_type IS NULL OR cve_desc IS NULL OR nvd_score IS NULL" + rowsAffected, err := ORM.Raw(sqlString, scoreType).QueryRows(&originExcel) + return originExcel, rowsAffected, err +} + +func SelectCveVulnCenter() ([]models.VulnCenter, int64, error) { + var vulnCenter []models.VulnCenter + sqlString := "SELECT cve_id, cve_num, repair_time FROM cve_vuln_center WHERE repair_time = '' OR repair_time IS NULL" + rowsAffected, err := ORM.Raw(sqlString).QueryRows(&vulnCenter) + return vulnCenter, rowsAffected, err +} + +func UpdateCveVulnCenter(repairTime, cveId string) error { + sqlString := "update cve_vuln_center set repair_time = ? where cve_id = ?" + _, err := ORM.Raw(sqlString, repairTime, cveId).Exec() + return err +} + +func QueryCveByNumber(cveNum string) ([]models.VulnCenter, int64, error) { + var vulnCenter []models.VulnCenter + sqlString := "select cve_num, pack_name, cve_version, cve_desc,repair_time,cve_status,cve_id from cve_vuln_center where cve_num = ? order by cve_id desc" + rowsAffected, err := ORM.Raw(sqlString, cveNum).QueryRows(&vulnCenter) + return vulnCenter, rowsAffected, err +} diff --git a/cve-vulner-manager/cve-timed-task/db_models/init_db.go b/cve-vulner-manager/cve-timed-task/db_models/init_db.go new file mode 100644 index 0000000..7692a99 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/db_models/init_db.go @@ -0,0 +1,13 @@ +package db_models + +import ( + "github.com/astaxie/beego/orm" + _ "github.com/go-sql-driver/mysql" +) + +var ORM orm.Ormer + +func InitTimedTaskDB() { + myOrm := orm.NewOrm() + ORM = myOrm +} diff --git a/cve-vulner-manager/cve-timed-task/deletetask/delete_file_task.go b/cve-vulner-manager/cve-timed-task/deletetask/delete_file_task.go new file mode 100644 index 0000000..7642919 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/deletetask/delete_file_task.go @@ -0,0 +1,48 @@ +package deletetask + +import ( + "github.com/astaxie/beego/logs" + "io/ioutil" + "os" + "time" +) + +func DelDir(filePath string) { + + fileList, err := ioutil.ReadDir(filePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + return + } + + for _, file := range fileList { + fileName := file.Name() + if file.IsDir() { + childFilePath := filePath + fileName + "/" + childFileList, err := ioutil.ReadDir(childFilePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + } + if len(childFileList) == 0 { + logs.Info("this is an empty folder", childFilePath) + err = os.Remove(childFilePath) + if err != nil { + logs.Error("remove empty folder filePath error:", childFilePath, err.Error()) + } + return + } + DelDir(childFilePath) + } + now := time.Now() + if now.Unix()-60*60*24*30 > file.ModTime().Unix() { + removePath := filePath + fileName + err = os.Remove(removePath) + if err != nil { + logs.Error("remove file removePath error:", err.Error()) + } + logs.Info(removePath, "is remove at", now) + } + + } + +} diff --git a/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go b/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go new file mode 100644 index 0000000..256a4a1 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go @@ -0,0 +1,316 @@ +package downloadfiletask + +import ( + "cvevulner/cve-timed-task/util" + "fmt" + "github.com/antchfx/htmlquery" + "github.com/astaxie/beego/logs" + "io" + "net/http" + "os" + "strings" +) + +func HandleOne() (string, error) { + filePath := "newexcels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/newexcels" + html, err := util.UrlToHTML(url) + if err != nil { + return "", err + } + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("xpath match error:", err) + return "", err + } + if len(names) == 0 { + break + } + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/newexcels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil + +} + +func HandleTwo() (string, error) { + filePath := "mappingexcels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/mappingexcels" + html, err := util.UrlToHTML(url) + if err != nil { + return "", err + } + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("xpath match error:", err) + return "", err + } + if len(names) == 0 { + break + } + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/mappingexcels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil +} + +func HandleThree() (string, error) { + filePath := "import_excels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/import_excels" + + html, err := util.UrlToHTML(url) + if err != nil { + return "", err + } + + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("xpath match error:", err) + return "", err + } + if len(names) == 0 { + break + } + + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/import_excels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + _ = resp.Body.Close() + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil +} + +func HandleFour() (string, error) { + filePath := "package_committer_excels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/package_committer_excels" + html, err := util.UrlToHTML(url) + if err != nil { + return "", err + } + + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("xpath match error:", err) + return "", err + } + if len(names) == 0 { + break + } + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/package_committer_excels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + _ = resp.Body.Close() + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil +} + +// DownloadSpecErrorExcels download spec_error_excels +func DownloadSpecErrorExcels() (string, error) { + filePath := "spec_error_excels/" + err := util.MakeDir(filePath) + if err != nil { + logs.Error("util.makeDir error", err) + return "", err + } + + url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/spec_error_excels" + html, err := util.UrlToHTML(url) + if err != nil { + return "", err + } + + for i := 3; ; i += 2 { + names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("xpath match error:", err) + return "", err + } + if len(names) == 0 { + break + } + name := htmlquery.SelectAttr(names[0], "title") + if strings.HasSuffix(name, ".xls") || strings.HasSuffix(name, ".xlsx") { + downloadUrl := fmt.Sprintf("https://gitee.com/openeuler/cve-manager/raw/master/cve-vulner-manager/cve-py/spec_error_excels/%s", name) + resp, err := http.Get(downloadUrl) + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + _ = resp.Body.Close() + continue + } + + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + _ = resp.Body.Close() + continue + } + + saveFile, err := os.Create(filePath + name) + if err != nil { + logs.Error("create file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy file error:", saveFile) + _ = resp.Body.Close() + _ = saveFile.Close() + continue + } + _ = resp.Body.Close() + _ = saveFile.Close() + } else { + logs.Warn("not Excel file") + } + } + return filePath, nil +} diff --git a/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go b/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go new file mode 100644 index 0000000..beebeed --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go @@ -0,0 +1,42 @@ +package emailtask + +import ( + "cvevulner/cve-timed-task/db_models" + "github.com/astaxie/beego/logs" + "io/ioutil" +) + +// Sending Query all recipients +func Sending(filePath string) { + result, _, err := db_models.GetCveEmailListByEmailType("1") + if err != nil { + logs.Error("db_models.GetCveEmailListByEmailType error:", err) + return + } + + fList := make([]string, 0, 10) + fileList, err := ioutil.ReadDir(filePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + } + + for _, file := range fileList { + if file.IsDir() { + continue + } + fileName := file.Name() + childFilePath := filePath + fileName + fList = append(fList, childFilePath) + } + + subject := "This time cve creates an issue record" + content := "The email attachment may contain two attached excel documents, which are the issue data that has been successfully created and the issue data that has not been created." + + for _, i := range result { + err = SendEmail([]string{i.EmailName}, []string{}, subject, content, fList) + if err != nil { + logs.Error("SendEmail error:", err) + } + } + +} diff --git a/cve-vulner-manager/cve-timed-task/emailtask/send_email.go b/cve-vulner-manager/cve-timed-task/emailtask/send_email.go new file mode 100644 index 0000000..fa41765 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/emailtask/send_email.go @@ -0,0 +1,41 @@ +package emailtask + +import ( + "github.com/astaxie/beego" + "gopkg.in/gomail.v2" + "strconv" +) + +func SendEmail(mailTo, mailCc []string, subject string, body string, filePath []string) error { + emailName := beego.AppConfig.String("email::email_name") + emailPwd := beego.AppConfig.String("email::email_pwd") + emailHost := beego.AppConfig.String("email::email_host") + emailPort := beego.AppConfig.String("email::email_port") + emailPwd = "" + + mailConn := map[string]string{ + "user": emailName, + "pass": emailPwd, + "host": emailHost, + "port": emailPort, + } + + port, _ := strconv.Atoi(mailConn["port"]) + m := gomail.NewMessage() + m.SetHeader("From", "cve-manager"+"<"+mailConn["user"]+">") + m.SetHeader("To", mailTo...) + if len(mailCc) > 0 { + m.SetHeader("Cc", mailCc...) + } + m.SetHeader("Subject", subject) + m.SetBody("text/plain", body) + + for _, v := range filePath { + m.Attach(v) + } + d := gomail.NewDialer(mailConn["host"], port, mailConn["user"], mailConn["pass"]) + err := d.DialAndSend(m) + + return err + +} diff --git a/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go b/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go new file mode 100644 index 0000000..576852b --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go @@ -0,0 +1,40 @@ +package emailtask + +import ( + "cvevulner/cve-timed-task/db_models" + "github.com/astaxie/beego/logs" + "io/ioutil" +) + +func SendingTypeTwo(filePath string) { + result, _, err := db_models.GetCveEmailListByEmailType("1") + if err != nil { + logs.Error("db_models.GetCveEmailListByEmailType error:", err) + return + } + + fList := make([]string, 0, 10) + fileList, err := ioutil.ReadDir(filePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + } + + for _, file := range fileList { + if file.IsDir() { + continue + } + fileName := file.Name() + childFilePath := filePath + fileName + fList = append(fList, childFilePath) + } + + subject := "Form error data feedback(人工CVE漏洞数据错误反馈)" + content := "The submitted CVE data is wrong, please check the format and content, please refer to the attachment" + + for _, i := range result { + err = SendEmail([]string{i.EmailName}, []string{}, subject, content, fList) + if err != nil { + logs.Error("SendEmail error:", err) + } + } +} diff --git a/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go b/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go new file mode 100644 index 0000000..18791c0 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go @@ -0,0 +1,137 @@ +package gitwebtask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "fmt" + "github.com/antchfx/htmlquery" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "strings" + "time" +) + +func Handle() { + url := "https://gitee.com/openeuler/community/tree/master/sig" + doc, err := util.UrlToHTML(url) + if err != nil { + logs.Error(" util.UrlToHTML error:", err, url) + return + } + + type sig struct { + sigName string + sigPage string + etherpad string + } + + sigsList := make([]sig, 0, 100) + for i := 3; ; i += 2 { + node, err := htmlquery.QueryAll(doc, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) + if err != nil { + logs.Error("htmlquery.QueryAll error:", err) + continue + } + if len(node) == 0 { + break + } + + sigName := htmlquery.SelectAttr(node[0], "title") + if sigName == "sigs.yaml" || sigName == "README.md" { + break + } + + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + _, err = db_models.GetOneGitRepoGroups(sigName) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + err = db_models.InsertOneGitRepoGroups(&models.GitRepoGroups{ + GroupName: sigName, + }) + if err != nil { + logs.Error("db_models.InsertOneGitRepoGroups error:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + } + } else { + logs.Error("db_models.InsertOneGitRepoGroups error:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + } + + sigPageNodes, err := htmlquery.QueryAll(doc, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@href", i)) + if err != nil { + logs.Warn("htmlquery.QueryAll error:", err) + } + sigPage := htmlquery.SelectAttr(sigPageNodes[0], "href") + + etherpad := fmt.Sprintf("https://etherpad.openeuler.org/p/%s-meetings", sigName) + s := sig{ + sigName: sigName, + sigPage: sigPage, + etherpad: etherpad, + } + //Get the name, homepage and etherpad of all sigs + sigsList = append(sigsList, s) + } + fmt.Println(sigsList) + for _, s := range sigsList { + gitRepoGroup, err := db_models.GetOneGitRepoGroups(s.sigName) + if err != nil { + logs.Warn("db_models.GetOneGitRepoGroups error:", err.Error()) + continue + } + err = db_models.DeleteGitRepoMember(gitRepoGroup.GroupId) + if err != nil { + logs.Warn("db_models.DeleteGitRepoMember error:", err.Error()) + continue + } + + urlOWNERS := fmt.Sprintf("https://gitee.com/openeuler/community/blob/master/sig/%s/OWNERS", s.sigName) + docOWNERS, err := util.UrlToHTML(urlOWNERS) + if err != nil { + logs.Error("util.UrlToHTML error:", err.Error(), url) + continue + } + nodeOWNERS, err := htmlquery.QueryAll(docOWNERS, "//div[@class=\"line\"]/text()") + if err != nil { + logs.Error("htmlquery.QueryAll error:", err) + continue + } + if len(nodeOWNERS) > 1 { + for _, k := range nodeOWNERS[1:] { + i := htmlquery.InnerText(k) + mSplit := strings.Split(i, "-") + maintainer := strings.TrimSpace(mSplit[len(mSplit)-1]) + giteRepoMember := &models.GiteRepoMember{ + GroupId: gitRepoGroup.GroupId, + MemberName: maintainer, + MemberType: "Maintainer", + CreateTime: time.Now().Format("2006-01-02 15:04:05"), + } + err = db_models.InsertOneGiteRepoMember(giteRepoMember) + if err != nil { + logs.Error("db_models.InsertOneGiteRepoMember error:", err.Error()) + continue + } + } + } + } + +} diff --git a/cve-vulner-manager/cve-timed-task/gitwebtask/yaml_task.go b/cve-vulner-manager/cve-timed-task/gitwebtask/yaml_task.go new file mode 100644 index 0000000..a75be2c --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/gitwebtask/yaml_task.go @@ -0,0 +1,6 @@ +package gitwebtask + +// DownLoad download detail +func DownLoad() { + +} diff --git a/cve-vulner-manager/cve-timed-task/start.go b/cve-vulner-manager/cve-timed-task/start.go new file mode 100644 index 0000000..39c24cb --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/start.go @@ -0,0 +1,17 @@ +package cve_timed_task + +import ( + "cvevulner/cve-timed-task/controller" + "cvevulner/cve-timed-task/db_models" +) + +func StartCveTimedTask() error { + + db_models.InitTimedTaskDB() + + err := controller.StartTimerTask() + if err != nil { + return err + } + return nil +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/common.go b/cve-vulner-manager/cve-timed-task/tabletask/common.go new file mode 100644 index 0000000..7bca973 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/common.go @@ -0,0 +1,66 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" +) + +// DeleteYamlData delete yaml data +func DeleteYamlData() { + packagesData, rowsAffected, err := db_models.SelectOpenEulerYamlData() + if err != nil { + logs.Error("db_models.SelectOpenEulerYamlData error:", err.Error()) + return + } + if rowsAffected > 0 { + for _, v := range packagesData { + if v.PackageId >= 10000000 { + _, errg := db_models.SelectGaussYamlOriginData(v.PackageName, v.Version) + _, errm := db_models.SelectMindSporeYamlOriginData(v.PackageName, v.Version) + _, erro := db_models.SelectOpenLookengYamlOriginData(v.PackageName, v.Version) + if errors.Is(orm.ErrNoRows, errg) && errors.Is(orm.ErrNoRows, errm) && errors.Is(orm.ErrNoRows, erro) { + logs.Info("ID of the currently deleted data: ", v.GitId) + //open transaction + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + err = db_models.DeleteYamlOpenEulerDetailData(v.GitId) + if err != nil { + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + err = db_models.DeleteYamlOpenEulerData(v.GitId) + if err != nil { + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } else { + if errg != nil { + logs.Error("db_models.SelectGaussYamlOriginData:", errg, v.PackageName, v.Version) + } + if errm != nil { + logs.Error("db_models.SelectMindSporeYamlOriginData", errm, v.PackageName, v.Version) + } + if erro != nil { + logs.Error("db_models.SelectOpenLookengYamlOriginData", erro, v.PackageName, v.Version) + } + } + } + } + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go new file mode 100644 index 0000000..e50ccd7 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go @@ -0,0 +1,306 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/util" + "github.com/antchfx/htmlquery" + "github.com/astaxie/beego/logs" + "github.com/pkg/errors" + "strings" + "time" +) + +type XpathList struct { + NvdScore string + CveLevel string + CveDesc string + RepairTime string + VectorValue string + AttackVector string + AccessVector string + AttackComplexity string + AccessComplexity string + PrivilegeRequired string + UserInteraction string + Scope string + Confidentiality string + Integrity string + Availability string + Authentication string + ScoreType string +} + +// Crawling Grab cve specific information +func Crawling(url string) (XpathList, error) { + logs.Info("now crawling :", url) + xpathList := XpathList{} + + if url == "" || !strings.Contains(url, "http") { + logs.Error("url is not standardized:", url) + return xpathList, errors.New("url is not standardized") + } + + html, err := util.UrlToHTML(url) + if err != nil { + logs.Error("util.UrlToHTML error:", err.Error(), url) + return xpathList, err + } + //xpathList.CveDesc + nodes, err := htmlquery.QueryAll(html, "//*[@id=\"vulnDetailTableView\"]//tr/td/div/div[1]/p[1]/text()") + if err != nil { + return xpathList, err + } else { + if len(nodes) > 0 { + cveDesc := htmlquery.InnerText(nodes[0]) + if cveDesc == "N/A" { + xpathList.CveDesc = "" + } else { + xpathList.CveDesc = cveDesc + } + } else { + logs.Info("xpath no match") + } + } + //xpathList.RepairTime + nodes, err = htmlquery.QueryAll(html, "//*[@data-testid=\"vuln-published-on\"]/text()") + if err != nil { + return xpathList, err + } else { + if len(nodes) > 0 { + repairTime := htmlquery.InnerText(nodes[0]) + if repairTime != "" { + if repairTime == "N/A" { + xpathList.RepairTime = "" + } else { + w, err := time.ParseInLocation("01/02/2006", repairTime, time.Local) + if err == nil { + xpathList.RepairTime = w.Format("2006-01-02 15:04:05") + } else { + logs.Error(err.Error()) + } + } + } + } else { + logs.Info("xpath no match") + } + } + xpathList.ScoreType = "v3.0" + + nodes, err = htmlquery.QueryAll(html, "//*[@id=\"nistV3MetricHidden\"]/@value") + if err != nil { + return xpathList, err + } + if len(nodes) == 0 { + nodes, err = htmlquery.QueryAll(html, "//*[@id=\"cnaV3MetricHidden\"]/@value") + if err != nil { + return xpathList, err + } + } + if len(nodes) > 0 { + value := htmlquery.SelectAttr(nodes[0], "value") + if value != "" { + childHtml, err := htmlquery.Parse(strings.NewReader(value)) + if err != nil { + return xpathList, err + } + + //xpathList.CveLevel + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-base-score-severity\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + cl := htmlquery.InnerText(nodes[0]) + if len(cl) > 0 { + cl = strings.TrimSpace(cl) + xpathList.CveLevel = strings.ToUpper(cl[:1]) + strings.ToLower(cl[1:]) + } + } + + //xpathList.NvdScore + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-base-score\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.NvdScore = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.VectorValue + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-vector\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + vv := htmlquery.InnerText(nodes[0]) + vv = strings.Replace(vv, "(", "", -1) + vv = strings.Replace(vv, ")", "", -1) + xpathList.VectorValue = strings.TrimSpace(vv) + } + //xpathList.AttackVector + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-av\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.AttackVector = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.AttackComplexity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-ac\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.AttackComplexity = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.PrivilegeRequired + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-pr\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.PrivilegeRequired = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.UserInteraction + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-ui\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.UserInteraction = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.Scope + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-s\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Scope = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.Confidentiality + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-c\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Confidentiality = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.Integrity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-i\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Integrity = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + //xpathList.Integrity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv3-a\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Availability = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + } + } else { + nodes, err = htmlquery.QueryAll(html, "//*[@id=\"nistV2MetricHidden\"]/@value") + if len(nodes) > 0 { + value := htmlquery.SelectAttr(nodes[0], "value") + if value != "" { + childHtml, err := htmlquery.Parse(strings.NewReader(value)) + if err != nil { + return xpathList, err + } + xpathList.ScoreType = "v2.0" + //xpathList.CveLevel + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-base-score-severity\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + cl := htmlquery.InnerText(nodes[0]) + if len(cl) > 0 { + cl = strings.TrimSpace(cl) + xpathList.CveLevel = strings.ToUpper(cl[:1]) + strings.ToLower(cl[1:]) + } + } + + //xpathList.NvdScore + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-base-score\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.NvdScore = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.VectorValue + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-vector\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + vv := htmlquery.InnerText(nodes[0]) + vv = strings.Replace(vv, "(", "", -1) + vv = strings.Replace(vv, ")", "", -1) + xpathList.VectorValue = strings.TrimSpace(vv) + } + + //xpathList.AccessVector + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-av\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.AccessVector = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.AccessComplexity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-ac\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.AccessComplexity = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.Authentication + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-au\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Authentication = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.Confidentiality + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-c\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Confidentiality = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.Integrity + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-i\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Integrity = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + + //xpathList.Availability + nodes, err = htmlquery.QueryAll(childHtml, "//*[@data-testid=\"vuln-cvssv2-a\"]/text()") + if err != nil { + return xpathList, err + } + if len(nodes) > 0 { + xpathList.Availability = strings.TrimSpace(htmlquery.InnerText(nodes[0])) + } + } + } + } + + return xpathList, nil +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go b/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go new file mode 100644 index 0000000..80cfbf9 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go @@ -0,0 +1,49 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/xuri/excelize/v2" +) + +func GenerateExcels(status, path string) { + results, rowsAffected, err := db_models.GetResults(status) + if err != nil { + logs.Error("db_models.GetResults error:", err) + return + } + if rowsAffected == 0 { + logs.Info("No issue record") + return + } + + f := excelize.NewFile() + index := f.NewSheet("Sheet1") + f.SetActiveSheet(index) + cc := []string{"id", "cve_id", "cve_num", "cve_desc", "cve_level", "cve_version", "repair_time", "pack_name", "nvd_score", "n_vector_value", "create_time", "update_time", "delete_time", "status", "error_description"} + rowBegin := "A1" + err = f.SetSheetRow("Sheet1", rowBegin, &cc) + if err != nil { + logs.Error("set row title error:", err) + return + } + + for k, v := range results { + rowBegin = fmt.Sprintf("A%d", k+2) + err := f.SetSheetRow("Sheet1", rowBegin, &[]interface{}{v.IssueRecordId, v.CveId, v.CveNum, v.Description, v.CveLevel, v.CveVersion, v.RepairTime, v.PackName, v.NVDScore, v.NvectorVule, v.CreateTime, v.UpdateTime, v.DeleteTime, v.Status, v.ErrorDescription}) + if err != nil { + logs.Error("f.SetSheetRow error:", err, rowBegin) + continue + } + } + err = f.SaveAs(path) + if err != nil { + logs.Error("f.SaveAs error: ", err, path) + } + + err = db_models.UpdateStatue(status) + if err != nil { + logs.Error("db_models.UpdateStatue error: ", err) + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go new file mode 100644 index 0000000..3def27b --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go @@ -0,0 +1,290 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "errors" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "io" + "net/http" + "os" + "time" +) + +// DownloadGuessYaml Download the yaml file of openGauss on Gitee +func DownloadGuessYaml() (string, error) { + filePath := "gauss_yaml/" + err := util.MakeDir(filePath) + if err != nil { + return "", err + } + yamlFileName := filePath + "gauss_yaml.yaml" + //download the yaml file + downloadUrl := "https://gitee.com/opengauss/openGauss-third_party/raw/master/Third_Party_Open_Source_Software_List.yaml" + resp, err := http.Get(downloadUrl) + defer resp.Body.Close() + if err != nil { + logs.Error("get", downloadUrl, "error", err.Error()) + return "", err + } + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + return "", errors.New(resp.Status) + } + saveFile, err := os.Create(yamlFileName) + defer saveFile.Close() + if err != nil { + logs.Error("Create file error", yamlFileName, err.Error()) + return "", err + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy resp.Body to file error", yamlFileName, err.Error()) + return "", err + } + return yamlFileName, nil +} + +func StoreYamlData(yamlData map[string]map[string]string) { + if len(yamlData) > 0 { + err := db_models.UpdateYamlOriginMark() + if err != nil { + logs.Error("db_models.UpdateYamlOriginMark error:", err.Error()) + return + } + for k, v := range yamlData { + if _, ok := v["version"]; !ok || v["version"] == "" { + v["version"] = "" + } + //open transaction + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + originData, err := db_models.SelectYamlOriginData(k, v["version"]) + now := time.Now() + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + logs.Warn("error is orm.ErrNoRows, Create a record") + openGuessYaml := &models.OpenGussYaml{ + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + Status: 1, + CpeName: v["cpeName"], + CreateTime: now.Format("2006-01-02 15:04:05"), + UpdateTime: now.Format("2006-01-02 15:04:05"), + MarkBit: 1, + Repo: "security", + } + err = db_models.InsertYamlOriginData(openGuessYaml) + if err != nil { + logs.Error("db_models.InsertYamlOriginData error:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + logs.Info("insert OpenGussYaml success") + } else { + logs.Error("db_models.SelectYamlOriginData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Info("database records, change the data") + openGuessYaml := &models.OpenGussYaml{ + Id: originData.Id, + OriginUrl: v["url"], + Status: 1, + CpeName: v["cpeName"], + UpdateTime: now.Format("2006-01-02 15:04:05"), + Repo: "security", + MarkBit: 1, + } + _, err = db_models.UpdateYamlOriginData(openGuessYaml) + if err != nil { + logs.Error("db_models.UpdateYamlOriginData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + packAgesData, rowsAffected, err := db_models.SelectYamlData(k, v["version"]) + if err != nil { + logs.Error("db_models.SelectYamlData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + if rowsAffected == 0 { + logs.Warn("No record found, Create a record") + var packId int64 + gitOpenEuler, err := db_models.SelectYamlLastData() + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + packId = 10000000 + } else { + logs.Error("db_models.SelectYamlLastData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + if gitOpenEuler.PackageId >= 10000000 { + packId = gitOpenEuler.PackageId + 1 + } else { + packId = 10000000 + } + } + goe := &models.GitOpenEuler{ + PackageId: packId, + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + CreateTime: now, + UpdateTime: now, + CpePackName: v["cpeName"], + Status: 1, + } + lastId, err := db_models.InsertYamlData(goe) + if err != nil { + logs.Error("db_models.InsertYamlData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + gpi := &models.GitPackageInfo{ + GitId: lastId, + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + CreateTime: now, + UpdateTime: now, + Decription: "", + Status: 0, + } + err = db_models.InsertYamlDetailData(gpi) + if err != nil { + logs.Error("db_models.InsertYamlDetailData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + } else { + if rowsAffected > 1 { + for _, pv := range packAgesData[1:] { + err = db_models.DeleteYamlOpenEulerDetailData(pv.GitId) + if err != nil { + logs.Error("db_models.DeleteYamlOpenEulerDetailData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + err = db_models.DeleteYamlOpenEulerData(pv.GitId) + if err != nil { + logs.Error("db_models.DeleteYamlOpenEulerData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } + goe := &models.GitOpenEuler{ + OriginUrl: v["url"], + UpdateTime: now, + CpePackName: v["cpeName"], + Status: 1, + PackageId: packAgesData[0].PackageId, + PackageName: k, + Version: v["version"], + } + err = db_models.UpdateYamlData(goe) + if err != nil { + logs.Error("db_models.UpdateYamlData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + gpi := &models.GitPackageInfo{ + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + UpdateTime: now, + Status: 0, + GitId: packAgesData[0].GitId, + } + err = db_models.UpdateYamlDetailData(gpi) + if err != nil { + logs.Error("db_models.UpdateYamlDetailData", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + logs.Info("Transaction committed successfully", k) + } + err = db_models.DeleteYamlOriginMark() + if err != nil { + logs.Error("db_models.DeleteYamlOriginMark error:", err.Error()) + } + } else { + logs.Warning("yamlData Is empty, nothing can be done") + } +} + +// ProcGaussYaml +//1. Create a folder +//2. Download yaml +//3. Parse yaml +//4. Save in mysql +func ProcGaussYaml() { + yamlFileName, err := DownloadGuessYaml() + if err != nil { + logs.Error("DownloadGuessYaml error:", err.Error()) + return + } + logs.Info("The download yaml file address is:", yamlFileName) + + yamlData, err := util.ParseYaml(yamlFileName) + if err != nil { + logs.Error("util.ParseYaml error:", yamlFileName, err) + return + } + StoreYamlData(yamlData) + + DeleteYamlData() + +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go b/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go new file mode 100644 index 0000000..ea56900 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go @@ -0,0 +1,194 @@ +package tabletask + +import ( + "crypto/sha1" + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "encoding/hex" + "errors" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/xuri/excelize/v2" + "io" + "os" + "strconv" + "time" +) + +func ImportDataTypeTwo() { + filePath, err := downloadfiletask.HandleThree() + if err != nil { + logs.Error("downloadfiletask.HandleThree error:", err) + return + } + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + f, err := os.Open(filePath + fileName) + if err != nil { + continue + } + hash := sha1.New() + _, err = io.Copy(hash, f) + if err != nil { + logs.Error("io.Copy :", err) + _ = f.Close() + continue + } + _ = f.Close() + + sum := hash.Sum(nil) + hashValue := hex.EncodeToString(sum) + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + if err != nil { + logs.Error("db_models.SelectCveFileHashByFileName :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + if rowsAffected > 0 && util.InSlice(list, hashValue) { + logs.Info("file has been parsed", fileName) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + _ = os.Remove(filePath + fileName) + continue + } + err = db_models.InsertCveFileHash(fileName, hashValue) + if err != nil { + logs.Error("db_models.InsertCveFileHash :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + logs.Error("excelize.OpenReader:", err) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + _ = excel.Close() + now := time.Now().Format("2006-01-02 15:04:05") + var packName string + var version string + var status int8 + var branchs string + for _, row := range rows[1:] { + if len(row) < 1 || row[0] == "" { + continue + } + packName = row[0] + if len(row) > 1 { + version = row[1] + if len(row) > 2 { + s, err := strconv.ParseInt(row[2], 10, 8) + if err != nil { + status = 1 + } else { + status = int8(s) + } + if len(row) > 3 { + branchs = row[3] + } else { + branchs = "" + } + } else { + status = 1 + } + } else { + version = "" + } + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + + _, err = db_models.SelectIssueRepoWhiteListByPackageName(row[0]) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + err = db_models.InsertIssueRepoWhiteList(&models.IssueRepoWhitelist{ + PackageName: packName, + Version: version, + Status: status, + Branchs: branchs, + CreateTime: now, + }) + if err != nil { + logs.Error("db_models.InsertIssueRepoWhiteList:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Error("db_models.SelectIssueRepoWhiteListByPackageName error", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + err = db_models.UpdateIssueRepoWhiteList(models.IssueRepoWhitelist{ + Status: status, + Branchs: branchs, + UpdateTime: now, + PackageName: packName, + Version: version, + }) + if err != nil { + logs.Error("db_models.UpdateIssueRepoWhiteList error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } + _ = excel.Close() + _ = os.Remove(filePath + fileName) + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go b/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go new file mode 100644 index 0000000..f2bcf91 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go @@ -0,0 +1,210 @@ +package tabletask + +import ( + "crypto/sha1" + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "encoding/hex" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "github.com/xuri/excelize/v2" + "io" + "os" + "time" +) + +// ToMysql Data stored in the database +func ToMysql() { + //filePath, err := downloadfiletask.HandleTwo() + //if err != nil { + // logs.Error("downloadfiletask.HandleTwo error:", err) + // return + //} + filePath := "mappingexcels/" + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + f, err := os.Open(filePath + fileName) + if err != nil { + _ = f.Close() + continue + } + + hash := sha1.New() + _, err = io.Copy(hash, f) + if err != nil { + logs.Error("io.Copy :", err) + _ = f.Close() + continue + } + _ = f.Close() + + sum := hash.Sum(nil) + hashValue := hex.EncodeToString(sum) + + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + if err != nil { + logs.Error("db_models.SelectCveFileHashByFileName :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + + if rowsAffected > 0 && util.InSlice(list, hashValue) { + logs.Info("file has been parsed", fileName) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + + _, err = os.Stat("oldexcels/") + if err != nil { + logs.Error("os.Stat error:", err.Error()) + if os.IsNotExist(err) { + logs.Info("And there is no such directory", filePath, "start to create") + err = os.Mkdir("oldexcels/", os.ModePerm) + if err != nil { + logs.Error("os.Mkdir error:", filePath, err.Error()) + } + logs.Info("create directory success:", filePath) + } + } + err = os.Rename(filePath+fileName, "oldexcels/"+fileName) + if err != nil { + logs.Error("os.Rename error:", filePath+fileName, "to", "oldexcels/"+fileName, err.Error()) + } + continue + } + + err = db_models.InsertCveFileHash(fileName, hashValue) + if err != nil { + logs.Error("db_models.InsertCveFileHash :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + logs.Error("excelize.OpenReader:", err, filePath+fileName) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + _ = excel.Close() + now := time.Now().Format("2006-01-02 15:04:05") + var packName string + var cpePackName string + for _, row := range rows[1:] { + if len(row) < 1 || row[0] == "" { + continue + } + packName = row[0] + + if len(row) > 1 { + cpePackName = row[1] + } else { + cpePackName = "" + } + + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + packageCpe, err := db_models.SelectCvePackAgeCpeByPackName(packName) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + err := db_models.InsertCvePackAgeCpe(&models.PackageCpe{ + PackName: packName, + CpePackName: cpePackName, + CreateTime: now, + }) + if err != nil { + logs.Error("db_models.InsertCvePackAgeCpe error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Error("db_models.SelectCvePackAgeCpeByPackName error: ", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + if packageCpe.CpePackName != cpePackName { + err := db_models.UpdateCvePackAgeCpe(models.PackageCpe{ + CpePackName: cpePackName, + CreateTime: now, + PackName: packageCpe.PackName, + }) + if err != nil { + logs.Error("db_models.UpdateCvePackAgeCpe error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } + _, err = os.Stat("oldexcels/") + if err != nil { + logs.Error("os.Stat error:", err.Error()) + if os.IsNotExist(err) { + logs.Info("And there is no such directory", filePath, "start to create") + err = os.Mkdir("oldexcels/", os.ModePerm) + if err != nil { + logs.Error("os.Mkdir error:", filePath, err.Error()) + } + logs.Info("create directory success:", filePath) + } + } + err = os.Rename(filePath+fileName, "oldexcels/"+fileName) + if err != nil { + logs.Error("os.Rename error:", filePath+fileName, "to", "oldexcels/"+fileName, err.Error()) + } + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go new file mode 100644 index 0000000..e9f611d --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go @@ -0,0 +1,291 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "errors" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "io" + "net/http" + "os" + "time" +) + +// DownloadGuessYaml Download the yaml file of openGauss on Gitee +func DownloadMindSporeYaml() (string, error) { + filePath := "mindspore_yaml/" + err := util.MakeDir(filePath) + if err != nil { + return "", err + } + yamlFileName := filePath + "mindspore_yaml.yaml" + //download the yaml file + downloadUrl := "https://gitee.com/mindspore/community/raw/master/security/config/Third_Party_Open_Source_Software_List.yaml" + resp, err := http.Get(downloadUrl) + defer resp.Body.Close() + if err != nil { + logs.Error("get", downloadUrl, "error", err.Error()) + return "", err + } + if resp.StatusCode != http.StatusOK { + logs.Error("get", downloadUrl, resp.Status) + return "", errors.New(resp.Status) + } + saveFile, err := os.Create(yamlFileName) + defer saveFile.Close() + if err != nil { + logs.Error("Create file error", yamlFileName, err.Error()) + return "", err + } + _, err = io.Copy(saveFile, resp.Body) + if err != nil { + logs.Error("copy resp.Body to file error", yamlFileName, err.Error()) + return "", err + } + return yamlFileName, nil +} + +// StoreOpenLookengYaml parse dict +//Store yaml data to mysql +func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { + if len(yamlData) > 0 { + err := db_models.UpdateMindYamlOriginMark() + if err != nil { + logs.Error("db_models.UpdateYamlOriginMarkLookeng:", err.Error()) + return + } + for RepoKey, RepoValue := range yamlData { + for k, v := range RepoValue { + if _, ok := v["version"]; !ok || v["version"] == "" { + v["version"] = "" + } + //open transaction + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + originData, err := db_models.SelectMindYamlOriginData(k, v["version"], RepoKey) + now := time.Now() + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + logs.Warn("error is orm.ErrNoRows, Create a record") + mindSporeYaml := &models.MindSporeYaml{ + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + Status: 1, + CpeName: v["cpeName"], + CreateTime: now.Format("2006-01-02 15:04:05"), + UpdateTime: now.Format("2006-01-02 15:04:05"), + MarkBit: 1, + Repo: RepoKey, + } + err = db_models.InsertMindYamlOriginData(mindSporeYaml) + if err != nil { + logs.Error("db_models.InsertMindYamlOriginData error:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + logs.Info("insert MindSporeYaml success") + } else { + logs.Error("db_models.SelectMindYamlOriginData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Info("database records, change the data") + mindSporeYaml := &models.MindSporeYaml{ + Id: originData.Id, + OriginUrl: v["url"], + Status: 1, + CpeName: v["cpeName"], + UpdateTime: now.Format("2006-01-02 15:04:05"), + MarkBit: 1, + } + _, err = db_models.UpdateMindYamlOriginData(mindSporeYaml) + if err != nil { + logs.Error("db_models.UpdateMindYamlOriginData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + packAgesData, rowsAffected, err := db_models.SelectYamlData(k, v["version"]) + if err != nil { + logs.Error("db_models.SelectMindYamlData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + if rowsAffected == 0 { + logs.Warn("No record found, Create a record") + var packId int64 + gitOpenEuler, err := db_models.SelectYamlLastData() + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + packId = 10000000 + } else { + logs.Error("db_models.SelectYamlLastData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + if gitOpenEuler.PackageId >= 10000000 { + packId = gitOpenEuler.PackageId + 1 + } else { + packId = 10000000 + } + } + goe := &models.GitOpenEuler{ + PackageId: packId, + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + CreateTime: now, + UpdateTime: now, + CpePackName: v["cpeName"], + Status: 1, + } + lastId, err := models.InsertYamlData(goe) + if err != nil { + logs.Error("db_models.InsertYamlData error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + gpi := &models.GitPackageInfo{ + GitId: lastId, + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + CreateTime: now, + UpdateTime: now, + Decription: "", + Status: 0, + } + err = db_models.InsertYamlDetailData(gpi) + if err != nil { + logs.Error("db_models.InsertYamlDetailData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", err.Error()) + } + continue + } + } else { + if rowsAffected > 1 { + for _, pv := range packAgesData[1:] { + err = db_models.DeleteYamlOpenEulerDetailData(pv.GitId) + if err != nil { + logs.Error("db_models.DeleteYamlOpenEulerDetailData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + err = db_models.DeleteYamlOpenEulerData(pv.GitId) + if err != nil { + logs.Error("db_models.DeleteYamlOpenEulerData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } + goe := &models.GitOpenEuler{ + OriginUrl: v["url"], + UpdateTime: now, + CpePackName: v["cpeName"], + Status: 1, + PackageId: packAgesData[0].PackageId, + PackageName: k, + Version: v["version"], + } + err = models.UpdateYamlData(goe) + if err != nil { + logs.Error("db_models.UpdateYamlData:", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + gpi := &models.GitPackageInfo{ + PackageName: k, + Version: v["version"], + OriginUrl: v["url"], + UpdateTime: now, + Status: 0, + GitId: packAgesData[0].GitId, + } + err = db_models.UpdateYamlDetailData(gpi) + if err != nil { + logs.Error("db_models.UpdateYamlDetailData", err.Error()) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + logs.Info("Transaction committed successfully", k) + } + } + err = db_models.DeleteMindYamlOriginMark() + if err != nil { + logs.Error("db_models.DeleteMindYamlOriginMark error:", err.Error()) + } + } else { + logs.Warning("yamlData Is empty, nothing can be done") + } + +} + +// ProcOpenLookengYaml +//1. Create a folder; +//2. Download yaml; +//3. Parse yaml; +//4. Save in mysql; +func ProcMindSporeYaml() { + yamlFileName, err := DownloadMindSporeYaml() + if err != nil { + logs.Error("DownloadMindSporeYaml error:", err.Error()) + return + } + yamlData, err := util.ParseYamlTypeTwo(yamlFileName) + if err != nil { + logs.Error("util.ParseYamlTypeTwo error: ", yamlFileName, err) + return + } + StoreMindSporeYaml(yamlData) + + DeleteYamlData() + +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go b/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go new file mode 100644 index 0000000..0c9d05a --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go @@ -0,0 +1,133 @@ +package tabletask + +import ( + "crypto/sha1" + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "encoding/hex" + "github.com/astaxie/beego/logs" + "github.com/xuri/excelize/v2" + "io" + "os" + "time" +) + +// import excel +func ImportData() { + filePath, err := downloadfiletask.HandleFour() + if err != nil { + logs.Error("downloadfiletask.HandleFour error:", err) + return + } + + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + f, err := os.Open(filePath + fileName) + if err != nil { + _ = f.Close() + continue + } + hash := sha1.New() + _, err = io.Copy(hash, f) + if err != nil { + logs.Error("io.Copy :", err) + _ = f.Close() + continue + } + _ = f.Close() + sum := hash.Sum(nil) + hashValue := hex.EncodeToString(sum) + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + if err != nil { + logs.Error("db_models.SelectCveFileHashByFileName :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + if rowsAffected > 0 && util.InSlice(list, hashValue) { + logs.Info("file has been parsed", fileName) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + _ = os.Remove(filePath + fileName) + continue + } + err = db_models.InsertCveFileHash(fileName, hashValue) + if err != nil { + logs.Error("db_models.InsertCveFileHash :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + _ = excel.Close() + logs.Error("excelize.OpenReader:", err) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + now := time.Now().Format("2006-01-02 15:04:05") + var packName string + var assignee string + for _, row := range rows[1:] { + if len(row) < 1 || row[0] == "" { + continue + } + packName = row[0] + + if len(row) > 1 { + assignee = row[1] + } else { + assignee = "" + } + err = db_models.InsertSpecIssueAssigness(&models.SpecIssueAssigness{ + PackageName: packName, + Assignee: assignee, + Status: 1, + CreateTime: now, + }) + if err != nil { + logs.Error(err) + continue + } + } + _ = excel.Close() + _ = os.Remove(filePath + fileName) + } + +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go b/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go new file mode 100644 index 0000000..8fec61a --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go @@ -0,0 +1,95 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/models" + "fmt" + "github.com/astaxie/beego/logs" + "strconv" + "time" +) + +// RepeatTask Parse the artificial CVE table, crawl the CVE official +// website data, and store it in the database +func RepeatTask() { + logs.Info("CVE with empty CVE official website data is re-crawled ") + + originExcel, rowsAffected, err := db_models.SelectCveOriginExcel("") + if err != nil { + logs.Error("db_models.SelectCveOriginExcel error:", err) + return + } + if rowsAffected == 0 { + return + } + fmt.Println(rowsAffected) + now := time.Now().Format("2006-01-02 15:04:05") + for _, v := range originExcel { + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", v.CveNum) + crawlList, err := Crawling(url) + if err != nil { + logs.Error("Crawling error:", err, url) + continue + } + nvdScore, err := strconv.ParseFloat(crawlList.NvdScore, 64) + if err != nil { + logs.Error(err.Error()) + } + err = db_models.UpdateCveOriginExcelTypeFour(models.OriginExcel{ + NVDScore: nvdScore, + CveLevel: crawlList.CveLevel, + CveDesc: crawlList.CveDesc, + RepairTime: crawlList.RepairTime, + NVDVector: crawlList.VectorValue, + AttackVector: crawlList.AttackVector, + AccessVector: crawlList.AccessVector, + AttackComplexity: crawlList.AttackComplexity, + AccessComplexity: crawlList.AccessComplexity, + PrivilegeRequired: crawlList.PrivilegeRequired, + UserInteraction: crawlList.UserInteraction, + Scope: crawlList.Scope, + Confidentiality: crawlList.Confidentiality, + Integrity: crawlList.Integrity, + Availability: crawlList.Availability, + Authentication: crawlList.Authentication, + CveStatus: 1, + UpdateTime: now, + ScoreType: crawlList.ScoreType, + CveNum: v.CveNum, + PackName: v.PackName, + CveVersion: v.CveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcelTypeFour error:", err) + } + } +} + +// GetPublishedDateTask Get the release date of nvd +func GetPublishedDateTask() { + logs.Info("Get the release date of nvd") + vulnCenter, rowsAffected, err := db_models.SelectCveVulnCenter() + if err != nil { + logs.Error("db_models.SelectCveVulnCenter error:", err) + return + } + if rowsAffected == 0 { + return + } + fmt.Println(rowsAffected) + for _, v := range vulnCenter { + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", v.CveNum) + crawlList, err := Crawling(url) + if err != nil { + logs.Error("Crawling error:", err, url) + continue + } + if crawlList.RepairTime != "" { + fmt.Println("upadd") + err := db_models.UpdateCveVulnCenter(crawlList.RepairTime, strconv.FormatInt(v.CveId, 10)) + if err != nil { + continue + } + } + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/run_task.go b/cve-vulner-manager/cve-timed-task/tabletask/run_task.go new file mode 100644 index 0000000..156b62c --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/run_task.go @@ -0,0 +1,317 @@ +package tabletask + +import ( + "crypto/sha1" + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "encoding/hex" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "github.com/xuri/excelize/v2" + "io" + "os" + "strconv" + "time" +) + +// HandleDate CVSS official website data crawling data storage database +func HandleDate() { + filePath, err := downloadfiletask.HandleOne() + if err != nil { + logs.Error("downloadfiletask.HandleOne error:", err) + return + } + + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + + f, err := os.Open(filePath + fileName) + if err != nil { + continue + } + hash := sha1.New() + _, err = io.Copy(hash, f) + if err != nil { + logs.Error("io.Copy :", err) + _ = f.Close() + continue + } + _ = f.Close() + sum := hash.Sum(nil) + hashValue := hex.EncodeToString(sum) + logs.Info("open transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + if err != nil { + logs.Error("db_models.SelectCveFileHashByFileName :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + fmt.Println(hashValue) + if rowsAffected > 0 && util.InSlice(list, hashValue) { + logs.Info("file has been parsed", fileName) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + _ = os.Remove(filePath + fileName) + continue + } + err = db_models.InsertCveFileHash(fileName, hashValue) + if err != nil { + logs.Error("db_models.InsertCveFileHash :", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + logs.Error("excelize.OpenReader:", err) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + _ = excel.Close() + + var packName string + var cveVersion string + var cveNum string + now := time.Now().Format("2006-01-02 15:04:05") + for _, row := range rows[1:] { + if len(row) < 1 { + continue + } + packName = row[0] + if len(row) > 1 { + cveVersion = row[1] + if len(row) > 2 { + cveNum = row[2] + } else { + cveNum = "" + } + } else { + cveVersion = "" + } + + logs.Info("begin transaction") + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + resultDict, err := db_models.GetCveOriginExcelTypeTwo(cveNum, packName, cveVersion) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + crawlList, err := Crawling(url) + if err != nil { + logs.Error("Crawling error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + nvdScore, err := strconv.ParseFloat(crawlList.NvdScore, 64) + if err != nil { + logs.Error(err.Error()) + } + err = db_models.InsertCveOriginExcel(&models.OriginExcel{ + CveNum: cveNum, + CveUrl: url, + CveVersion: cveVersion, + PackName: packName, + ScoreType: crawlList.ScoreType, + NVDScore: nvdScore, + CveLevel: crawlList.CveLevel, + CveDesc: crawlList.CveDesc, + RepairTime: crawlList.RepairTime, + NVDVector: crawlList.VectorValue, + AttackVector: crawlList.AttackVector, + AccessVector: crawlList.AccessVector, + AttackComplexity: crawlList.AttackComplexity, + AccessComplexity: crawlList.AccessComplexity, + PrivilegeRequired: crawlList.PrivilegeRequired, + UserInteraction: crawlList.UserInteraction, + Scope: crawlList.Scope, + Confidentiality: crawlList.Confidentiality, + Integrity: crawlList.Integrity, + Availability: crawlList.Availability, + Authentication: crawlList.Authentication, + CveStatus: 0, + CreateTime: now, + UpdateTime: now, + }) + if err != nil { + logs.Error("db_models.InsertCveOriginExcel error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + + } else { + logs.Error("db_models.GetCveOriginExcelTypeTwo error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + crawlList, err := Crawling(url) + if err != nil { + logs.Error("Crawling error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + if crawlList.NvdScore != "" || crawlList.CveDesc != "" { + nvdScore, err := strconv.ParseFloat(crawlList.NvdScore, 64) + if err != nil { + logs.Error(err.Error()) + } + if crawlList.CveDesc != "" && resultDict.NVDScore == nvdScore && resultDict.NVDVector == crawlList.VectorValue { + comp := make([]interface{}, 0, 10) + comp = append(comp, 3, 4, 7) + if util.InSlice(comp, int(resultDict.CveStatus)) { + logs.Info("update data aa:", cveNum) + err = db_models.UpdateCveOriginExcelTypeThree(models.OriginExcel{ + NVDScore: nvdScore, + CveLevel: crawlList.CveLevel, + CveDesc: crawlList.CveDesc, + RepairTime: crawlList.RepairTime, + NVDVector: crawlList.VectorValue, + AttackVector: crawlList.AttackVector, + AccessVector: crawlList.AccessVector, + AttackComplexity: crawlList.AttackComplexity, + AccessComplexity: crawlList.AccessComplexity, + PrivilegeRequired: crawlList.PrivilegeRequired, + UserInteraction: crawlList.UserInteraction, + Scope: crawlList.Scope, + Confidentiality: crawlList.Confidentiality, + Integrity: crawlList.Integrity, + Availability: crawlList.Availability, + Authentication: crawlList.Authentication, + CveStatus: 1, + UpdateTime: now, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcelTypeThree error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Info("The database is the latest data:", cveNum) + } + } else { + logs.Info("update data dd:", cveNum) + err = db_models.UpdateCveOriginExcelTypeFour(models.OriginExcel{ + NVDScore: nvdScore, + CveLevel: crawlList.CveLevel, + CveDesc: crawlList.CveDesc, + RepairTime: crawlList.RepairTime, + NVDVector: crawlList.VectorValue, + AttackVector: crawlList.AttackVector, + AccessVector: crawlList.AccessVector, + AttackComplexity: crawlList.AttackComplexity, + AccessComplexity: crawlList.AccessComplexity, + PrivilegeRequired: crawlList.PrivilegeRequired, + UserInteraction: crawlList.UserInteraction, + Scope: crawlList.Scope, + Confidentiality: crawlList.Confidentiality, + Integrity: crawlList.Integrity, + Availability: crawlList.Availability, + Authentication: crawlList.Authentication, + CveStatus: 1, + UpdateTime: now, + ScoreType: crawlList.ScoreType, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcelTypeFour error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } else { + logs.Error("error:", resultDict) + } + } + + specError, err := db_models.GetCveSpecError(cveNum) + if err != nil { + if !errors.Is(orm.ErrNoRows, err) { + logs.Error("db_models.GetCveSpecError error:", err) + } + } else { + err = db_models.UpdateCveOriginExcelTypeFive(models.OriginExcel{ + CveDesc: specError.Description, + CveStatus: 6, + CveNum: cveNum, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcelTypeFive error:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error("Rollback transaction error:", tranErr.Error()) + } + continue + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go b/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go new file mode 100644 index 0000000..6f25f90 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go @@ -0,0 +1,136 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" + "cvevulner/models" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "github.com/xuri/excelize/v2" + "os" + "time" +) + +// AddErrorDetails Conditions for filtering incorrect data +func AddErrorDetails() { + filePath, err := downloadfiletask.DownloadSpecErrorExcels() + if err != nil { + logs.Error("downloadfiletask.DownloadSpecErrorExcels error:", err) + return + } + + files, err := os.ReadDir(filePath) + if err != nil { + logs.Error(" os.ReadDir error: ", err, filePath) + return + } + + for _, file := range files { + fileName := file.Name() + if file.IsDir() { + logs.Warn(filePath+fileName, "is Dir, remove") + _ = os.RemoveAll(filePath + fileName) + continue + } + excel, err := excelize.OpenFile(filePath + fileName) + if err != nil { + logs.Error("excelize.OpenReader:", err) + continue + } + + rows, err := excel.GetRows("Sheet1") + if err != nil { + logs.Error("excel.GetRows Sheet1 error", err) + _ = excel.Close() + continue + } + _ = excel.Close() + var cveNum string + var cveDesc string + var cveOwner string + var packName string + now := time.Now().Format("2006-01-02 15:04:05") + for _, row := range rows[1:] { + if len(row) < 2 { + continue + } + cveNum = row[1] + if len(row) > 2 { + cveOwner = row[2] + if len(row) > 3 { + packName = row[3] + if len(row) > 4 { + cveDesc = row[4] + } else { + cveDesc = "" + } + } else { + packName = "" + } + } else { + cveOwner = "" + } + tranErr := db_models.ORM.Begin() + if tranErr != nil { + logs.Error(" Open transaction error:", tranErr.Error()) + continue + } + specError, err := db_models.GetCveSpecError(cveNum) + if err != nil { + if errors.Is(orm.ErrNoRows, err) { + _, err = db_models.InsertCveSpecError(&models.SpecError{ + CveNum: cveNum, + Description: cveDesc, + Owner: cveOwner, + Status: 1, + PackName: packName, + CreateTime: now, + }) + if err != nil { + logs.Error("db_models.InsertCveSpecError:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Error("db_models.GetCveSpecError:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } else { + logs.Info("CVE recorded", cveNum) + if specError.Description != cveDesc || specError.Owner != cveOwner || specError.PackName != packName || specError.Status != 1 { + logs.Info("update cve_spec_error") + err := db_models.UpdateCveSpecError(models.SpecError{ + CveNum: specError.CveNum, + Description: cveDesc, + Owner: cveOwner, + PackName: packName, + Status: 1, + UpdateTime: now, + }) + if err != nil { + logs.Error("db_models.UpdateCveSpecError:", err) + tranErr = db_models.ORM.Rollback() + if tranErr != nil { + logs.Error(" Rollback transaction error:", tranErr.Error()) + } + continue + } + } + } + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + continue + } + } + _ = os.Remove(filePath + fileName) + } +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go new file mode 100644 index 0000000..1e64a89 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go @@ -0,0 +1,393 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/util" + "cvevulner/models" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" + "github.com/pkg/errors" + "strconv" + "strings" + "time" +) + +//var cveListDate []db_models.VulnCenter + +type Compare struct { + CveId string + CveNum string +} + +// QueryCveALLData Find cve with missing necessary fields +func QueryCveALLData(days int) []models.VulnCenter { + now := time.Now() + beforeDate := time.Date(now.Year(), now.Month(), now.Day()-days, 0, 0, 0, 0, now.Location()).Format("2006-01-02 15:04:05") + + cveIdList := make([]Compare, 0, 100) + scoreList, rowsAffected, err := db_models.GetCveScoreListByBeforeDate(beforeDate) + if err == nil { + if rowsAffected > 0 { + for _, v := range scoreList { + cveIdList = append(cveIdList, Compare{ + CveId: strconv.FormatInt(v.CveId, 10), + CveNum: v.CveNum, + }) + } + } else { + logs.Info("No query results") + } + } else { + logs.Error("db_models.GetCveScoreList error:", err.Error()) + } + vulnCenterList, rowsAffected, err := db_models.GetCveVulnCenterList("", "", beforeDate) + if err == nil { + if rowsAffected > 0 { + for _, v := range vulnCenterList { + cveIdList = append(cveIdList, Compare{ + CveId: strconv.FormatInt(v.CveId, 10), + CveNum: v.CveNum, + }) + } + } else { + logs.Info("No query results") + } + } else { + logs.Error("db_models.GetCveVulnCenter error:", err.Error()) + } + + issueTemplateList, rowsAffected, err := db_models.GetCveIssueTemplate("0", "", "", beforeDate) + if err == nil { + if rowsAffected > 0 { + for _, v := range issueTemplateList { + cveIdList = append(cveIdList, Compare{ + CveId: strconv.FormatInt(v.CveId, 10), + CveNum: v.CveNum, + }) + } + } else { + logs.Info("No query results") + } + + } else { + logs.Error("db_models.GetCveIssueTemplate error:", err.Error()) + } + + cveIdList = RemoveRepeatedElement(cveIdList) + cveList := make([]models.VulnCenter, 0, len(cveIdList)) + if len(cveIdList) > 0 { + for _, v := range cveIdList { + vulnCenter, err := db_models.GetOneVulnCenter(v.CveId, v.CveNum) + if err != nil { + logs.Error("db_models.GetOneVulnCenter error:", err.Error()) + continue + } + + cveList = append(cveList, vulnCenter) + } + } + return cveList +} + +func RemoveRepeatedElement(arr []Compare) (newArr []Compare) { + newArr = make([]Compare, 0) + for i := 0; i < len(arr); i++ { + repeat := false + for j := i + 1; j < len(arr); j++ { + if arr[i] == arr[j] { + repeat = true + break + } + } + if !repeat { + newArr = append(newArr, arr[i]) + } + } + return +} + +// UpdateCveVuln update data +func UpdateCveVuln(url string, cve models.VulnCenter) error { + cveNum := strings.TrimSpace(cve.CveNum) + cveId := cve.CveId + cveStatus := cve.Status + cveDesc := strings.TrimSpace(cve.Description) + repairTime := strings.TrimSpace(cve.RepairTime) + updateTime := time.Now() + + cveStatusx := cveStatus + comp := make([]interface{}, 0, 10) + comp = append(comp, 0, 3, 4, 5, 6, 7, 8) + if util.InSlice(comp, int(cveStatus)) { + cveStatusx = 0 + } + compTwo := make([]interface{}, 0, 10) + compTwo = append(compTwo, 1, 2) + if util.InSlice(compTwo, int(cveStatus)) { + cveStatusx = 1 + } + listx, err := Crawling(url) + fmt.Println(listx) + if err != nil { + logs.Error("crawling data error:", err) + return err + } + + if (listx.CveDesc == "" || len(listx.CveDesc) < 2) && (listx.NvdScore == "") && (listx.RepairTime == "" || len(listx.RepairTime) > 2) { + logs.Info("The data does not exist and will not be processed temporarily", listx) + return errors.New("The data does not exist and will not be processed temporarily") + } + + if repairTime == "" && listx.RepairTime != "" && len(listx.RepairTime) > 2 { + if len(listx.RepairTime) > 10 { + repairTime = listx.RepairTime[:10] + } else { + repairTime = listx.RepairTime + } + err = db_models.UpdateVulnCenterTypeOne(models.VulnCenter{ + RepairTime: repairTime, + UpdateTime: updateTime, + Status: cveStatusx, + CveId: cveId, + }) + if err != nil { + logs.Error("db_models.UpdateVulnCenterTypeOne error:", err) + return err + } + } + + if listx.CveDesc != "" && len(listx.CveDesc) > 2 && cveDesc == "" { + err = db_models.UpdateVulnCenterTypeTwo(models.VulnCenter{ + Description: listx.CveDesc, + UpdateTime: updateTime, + Status: cveStatusx, + CveId: cveId, + }) + if err != nil { + logs.Error("db_models.UpdateVulnCenterTypeTwo error:", err) + return err + } + } + + nvdScore, err := strconv.ParseFloat(listx.NvdScore, 64) + if err != nil { + logs.Error(err.Error()) + } + if nvdScore > 0 && listx.NvdScore != "" { + var scoreType string + if listx.ScoreType == "v3.0" { + scoreType = "v3" + } else { + scoreType = "v2" + } + score, err := db_models.GetCveScoreByCveIdAndCveNum(cveId, cveNum) + if err != nil { + if errors.Is(err, orm.ErrNoRows) { + logs.Info(err) + } else { + logs.Error("db_models.GetCveScoreByCveIdAndCveNum error:", err) + return err + } + } else { + if score.NVDScore == 0 { + fmt.Println("up2") + err := db_models.UpdateCveScore(models.Score{ + NVDScore: nvdScore, + NvectorVule: listx.VectorValue, + NattackVector: listx.AttackVector, + NaccessVector: listx.AccessVector, + NattackComplexity: listx.AttackComplexity, + NaccessComplexity: listx.AccessComplexity, + NprivilegeRequired: listx.PrivilegeRequired, + NuserInteraction: listx.UserInteraction, + Nscope: listx.Scope, + Nconfidentiality: listx.Confidentiality, + Nintegrity: listx.Integrity, + Navailability: listx.Availability, + Nauthentication: listx.Authentication, + UpdateTime: updateTime, + ScoreType: scoreType, + Id: score.Id, + }) + if err != nil { + logs.Error("db_models.UpdateCveScore error:", err) + return err + } + } + } + } + + templateResult, err := db_models.GetIssueTemplateTypeOne(models.IssueTemplate{ + NVDScore: 0, + NVDVector: "", + CveBrief: "", + CveId: cveId, + CveNum: cveNum, + }) + if err != nil { + logs.Error(" db_models.GetIssueTemplateTypeOne error:", err) + return nil + } + + flag := false + if templateResult.NVDScore == 0 && listx.NvdScore != "" && nvdScore > 0 { + err = db_models.UpdateCveIssueTemplateTypeOne(models.IssueTemplate{ + NVDScore: nvdScore, + UpdateTime: updateTime, + TemplateId: templateResult.TemplateId, + }) + if err != nil { + logs.Error("db_models.UpdateCveIssueTemplateTypeOne error:", err) + return err + } + flag = true + } + + if (templateResult.NVDVector == "" || len(templateResult.NVDVector) < 2) && (listx.VectorValue != "" && len(listx.VectorValue) > 2) { + err = db_models.UpdateCveIssueTemplateTypeTwo(models.IssueTemplate{ + NVDVector: listx.VectorValue, + UpdateTime: updateTime, + TemplateId: templateResult.TemplateId, + }) + if err != nil { + logs.Error("db_models.UpdateCveIssueTemplateTypeTwo error:", err) + return err + } + flag = true + } + + if (templateResult.CveBrief == "" || len(templateResult.CveBrief) < 2) && (listx.CveDesc != "" && len(listx.CveDesc) > 2) { + err = db_models.UpdateCveIssueTemplateTypeThree(models.IssueTemplate{ + CveBrief: listx.CveDesc, + UpdateTime: updateTime, + TemplateId: templateResult.TemplateId, + }) + if err != nil { + logs.Error("db_models.UpdateCveIssueTemplateTypeThree error:", err) + return err + } + flag = true + } + + if flag { + err = db_models.UpdateVulnCenter(models.VulnCenter{ + UpdateTime: updateTime, + Status: cveStatusx, + CveId: cveId, + }) + if err != nil { + logs.Error("db_models.UpdateVulnCenter error:", err) + return err + } + } + return nil +} + +// SupplementCve +//1. Find cve with missing necessary fields; +//2. Go to the CVE official website to find the CVE information; +//3. Fill in the corresponding table again; +func SupplementCve() { + + cveList := QueryCveALLData(30) + + if len(cveList) > 0 { + for _, cve := range cveList { + cveNum := strings.TrimSpace(cve.CveNum) + cveVersion := strings.TrimSpace(cve.CveVersion) + packName := strings.TrimSpace(cve.PackName) + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + _ = UpdateCveVuln(url, cve) + resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName) + if err != nil { + logs.Error("db_models.GetSpceError error:", err) + continue + } + logs.Info("Filter, modify the status to 6", cveNum) + err = db_models.UpdateCveOriginExcel(models.OriginExcel{ + CveDesc: resultSpecError.Description, + CveStatus: 6, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcel error:", err) + } + } + } +} + +// LongSupplementCve 1. Find cve with missing necessary fields; +//2. Go to the CVE official website to find the CVE information; +//3. Fill in the corresponding table again; +func LongSupplementCve() { + cveList := QueryCveALLData(1500) + if len(cveList) > 0 { + for _, cve := range cveList { + cveNum := strings.TrimSpace(cve.CveNum) + cveVersion := strings.TrimSpace(cve.CveVersion) + packName := strings.TrimSpace(cve.PackName) + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + _ = UpdateCveVuln(url, cve) + resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName) + if err != nil { + logs.Error("db_models.GetSpceError error:", err) + continue + } + logs.Info("Filter, modify the status to 6", cveNum) + err = db_models.UpdateCveOriginExcel(models.OriginExcel{ + CveDesc: resultSpecError.Description, + CveStatus: 6, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcel error:", err) + } + } + } +} + +func PullCve(cveNum string) (resp int) { + resp = 0 + cveList, rowsAffected, err := db_models.QueryCveByNumber(cveNum) + if err != nil { + return resp + } + if rowsAffected == 0 { + return resp + } + fmt.Println(rowsAffected) + + if len(cveList) > 0 { + for _, cve := range cveList { + cveNum := strings.TrimSpace(cve.CveNum) + cveVersion := strings.TrimSpace(cve.CveVersion) + packName := strings.TrimSpace(cve.PackName) + url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) + err = UpdateCveVuln(url, cve) + if err == nil { + resp = 1 + } + resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName) + if err != nil { + logs.Error("db_models.GetSpceError error:", err) + } + logs.Info("Filter, modify the status to 6", cveNum) + err = db_models.UpdateCveOriginExcel(models.OriginExcel{ + CveDesc: resultSpecError.Description, + CveStatus: 6, + CveNum: cveNum, + PackName: packName, + CveVersion: cveVersion, + }) + if err != nil { + logs.Error("db_models.UpdateCveOriginExcel error:", err) + } + } + } + return resp +} diff --git a/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go b/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go new file mode 100644 index 0000000..b3cc56e --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go @@ -0,0 +1,52 @@ +package tabletask + +import ( + "cvevulner/cve-timed-task/db_models" + "fmt" + "github.com/astaxie/beego/logs" + "github.com/xuri/excelize/v2" +) + +func GenerateTable(path string) { + results, rowsAffected, err := db_models.GetCveOriginExcel() + if err != nil { + logs.Error("db_models.GetCveOriginExcel error:", err) + return + } + if rowsAffected == 0 { + logs.Info("The query result is empty, no error data") + return + } + + f := excelize.NewFile() + index := f.NewSheet("Sheet1") + f.SetActiveSheet(index) + + cc := []string{"cve_id", "cve_num", "cve_url", "cve_version", "pack_name", "score_type", "nvd_score", "cve_level", "cve_desc", "repair_time", "vector_value", "attack_vector", "access_vector", "attack_complexity", "access_complexity", "privilege_required", "user_interaction", "scope", "confidentiality", "integrity", "availability", "authentication", "cve_status", "create_time", "update_time", "delete_time"} + rowBegin := "A1" + err = f.SetSheetRow("Sheet1", rowBegin, &cc) + if err != nil { + logs.Error("set row title error:", err) + return + } + + for k, v := range results { + rowBegin = fmt.Sprintf("A%d", k+2) + err := f.SetSheetRow("Sheet1", rowBegin, &[]interface{}{v.CveId, v.CveNum, v.CveUrl, v.CveVersion, v.PackName, v.ScoreType, v.NVDScore, v.CveLevel, v.CveDesc, v.RepairTime, v.NVDVector, v.AttackVector, v.AccessVector, v.AttackComplexity, v.AccessComplexity, v.PrivilegeRequired, v.UserInteraction, v.Scope, v.Confidentiality, v.Integrity, v.Availability, v.Authentication, v.CveStatus, v.CreateTime, v.UpdateTime, v.UpdateTime}) + if err != nil { + logs.Error("f.SetSheetRow error:", err, rowBegin) + continue + } + } + + err = f.SaveAs(path) + if err != nil { + logs.Error("f.SaveAs error: ", err, path) + } + + err = db_models.UpdateStatusTypeTwo() + if err != nil { + logs.Error("db_models.UpdateStatusTypeTwo error: ", err) + } + +} diff --git a/cve-vulner-manager/cve-timed-task/util/crawl.go b/cve-vulner-manager/cve-timed-task/util/crawl.go new file mode 100644 index 0000000..7f146b7 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/util/crawl.go @@ -0,0 +1,33 @@ +package util + +import ( + "github.com/astaxie/beego/logs" + "github.com/pkg/errors" + "golang.org/x/net/html" + "golang.org/x/net/html/charset" + "net/http" +) + +func UrlToHTML(url string) (*html.Node, error) { + resp, err := http.Get(url) + defer resp.Body.Close() + if err != nil { + logs.Error("GET", url, "error:", err.Error()) + return nil, err + } + if resp.StatusCode != http.StatusOK { + logs.Error("response status code is:", resp.Status) + return nil, errors.New(resp.Status) + } + r, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type")) + if err != nil { + logs.Error("charset.NewReader error:", err.Error()) + return nil, err + } + doc, err := html.Parse(r) + if err != nil { + logs.Error("html.Parse error:", err.Error()) + return nil, err + } + return doc, nil +} diff --git a/cve-vulner-manager/cve-timed-task/util/deduplication.go b/cve-vulner-manager/cve-timed-task/util/deduplication.go new file mode 100644 index 0000000..b8ab057 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/util/deduplication.go @@ -0,0 +1,10 @@ +package util + +func InSlice(list []interface{}, item interface{}) bool { + for _, eachItem := range list { + if item == eachItem { + return true + } + } + return false +} diff --git a/cve-vulner-manager/cve-timed-task/util/makrdir.go b/cve-vulner-manager/cve-timed-task/util/makrdir.go new file mode 100644 index 0000000..c771d22 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/util/makrdir.go @@ -0,0 +1,53 @@ +package util + +import ( + "github.com/astaxie/beego/logs" + "io/ioutil" + "os" +) + +// MakeDir Create a directory,if there has this directory, delete all files in this directory +func MakeDir(filePath string) error { + _, err := os.Stat(filePath) + if err != nil { + logs.Error("os.Stat error:", err.Error()) + if os.IsNotExist(err) { + logs.Info("And there is no such directory", filePath, "start to create") + err = os.Mkdir(filePath, os.ModePerm) + if err != nil { + logs.Error("os.Mkdir error:", filePath, err.Error()) + return err + } + logs.Info("create directory success:", filePath) + return nil + } + return err + } + + fileList, err := ioutil.ReadDir(filePath) + if err != nil { + logs.Error("ReadDir error:", filePath, err.Error()) + return err + } + + for _, file := range fileList { + fileName := file.Name() + childFilePath := filePath + fileName + if file.IsDir() { + err = os.RemoveAll(childFilePath) + if err != nil { + logs.Error("os.RemoveAll error:", childFilePath, err.Error()) + return err + } + logs.Info("directory deleted successfully") + } else { + err = os.Remove(childFilePath) + if err != nil { + logs.Error("os.Remove error:", childFilePath, err.Error()) + return err + } + logs.Info("Delete file successfully") + } + } + return nil +} diff --git a/cve-vulner-manager/cve-timed-task/util/parse_yaml.go b/cve-vulner-manager/cve-timed-task/util/parse_yaml.go new file mode 100644 index 0000000..7695120 --- /dev/null +++ b/cve-vulner-manager/cve-timed-task/util/parse_yaml.go @@ -0,0 +1,42 @@ +package util + +import ( + "github.com/astaxie/beego/logs" + "gopkg.in/yaml.v2" + "io/ioutil" +) + +// ParseYaml Parse yaml +func ParseYaml(filePath string) (map[string]map[string]string, error) { + file, err := ioutil.ReadFile(filePath) + if err != nil { + logs.Error("ioutil.ReadFile error:", filePath, err.Error()) + return nil, err + } + + resultMap := make(map[string]map[string]string) + err = yaml.Unmarshal(file, resultMap) + if err != nil { + logs.Error("yaml.Unmarshal error:", err) + return nil, err + } + + return resultMap, nil +} + +func ParseYamlTypeTwo(filePath string) (map[string]map[string]map[string]string, error) { + file, err := ioutil.ReadFile(filePath) + if err != nil { + logs.Error("ioutil.ReadFile error:", filePath, err.Error()) + return nil, err + } + resultMap := make(map[string]map[string]map[string]string) + + err = yaml.Unmarshal(file, resultMap) + if err != nil { + logs.Error("yaml.Unmarshal error:", err) + return nil, err + } + return resultMap, nil + +} diff --git a/cve-vulner-manager/go.mod b/cve-vulner-manager/go.mod index 879795c..16aab38 100644 --- a/cve-vulner-manager/go.mod +++ b/cve-vulner-manager/go.mod @@ -3,34 +3,20 @@ module cvevulner go 1.14 require ( - gitee.com/openeuler/go-gitee v0.0.0-20210201095646-fd9179920723 github.com/360EntSecGroup-Skylar/excelize/v2 v2.3.0 + github.com/antchfx/htmlquery v1.2.4 github.com/astaxie/beego v1.12.3 - github.com/beego/bee v1.12.3 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible - github.com/flosch/pongo2 v0.0.0-20200805083417-63c99409991d // indirect github.com/go-sql-driver/mysql v1.5.0 github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect github.com/lib/pq v1.8.0 // indirect - github.com/magiconair/properties v1.8.4 // indirect - github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect - github.com/mitchellh/mapstructure v1.3.3 // indirect - github.com/pelletier/go-toml v1.8.1 // indirect - github.com/peterh/liner v1.2.0 // indirect - github.com/sirupsen/logrus v1.7.0 // indirect + github.com/pkg/errors v0.9.1 + github.com/robfig/cron/v3 v3.0.1 github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect github.com/smartystreets/goconvey v1.6.4 - github.com/spf13/afero v1.4.1 // indirect - github.com/spf13/cast v1.3.1 // indirect - github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect - github.com/spf13/viper v1.7.1 // indirect - go.starlark.net v0.0.0-20201113214410-e292e66a28cd // indirect - golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff // indirect - golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7 // indirect - golang.org/x/text v0.3.4 // indirect + github.com/xuri/excelize/v2 v2.5.0 + golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 + gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df - gopkg.in/ini.v1 v1.62.0 // indirect - gopkg.in/yaml.v2 v2.3.0 // indirect + gopkg.in/yaml.v2 v2.3.0 ) diff --git a/cve-vulner-manager/go.sum b/cve-vulner-manager/go.sum index a2f4c23..7b21725 100644 --- a/cve-vulner-manager/go.sum +++ b/cve-vulner-manager/go.sum @@ -1,75 +1,33 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -gitee.com/openeuler/go-gitee v0.0.0-20210201095646-fd9179920723 h1:awXn0FlbiQ9Ohj96FrWM1r16tB//D+eTyC2wainPX80= -gitee.com/openeuler/go-gitee v0.0.0-20210201095646-fd9179920723/go.mod h1:TQrS/LP/DFXLqM+lVrZd4nL2pbTrqiXABGT9PJepVTA= github.com/360EntSecGroup-Skylar/excelize/v2 v2.3.0 h1:tDWYNCJrpNnlNg8mVdlzAzPjlPaRbsA/kS8H9LczleQ= github.com/360EntSecGroup-Skylar/excelize/v2 v2.3.0/go.mod h1:Uwb0d1GgxJieUWZG5WylTrgQ2SrldfjagAxheU8W6MQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Knetic/govaluate v3.0.0+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/OwnLocal/goes v1.0.0/go.mod h1:8rIFjBGTue3lCU0wplczcUgt9Gxgrkkrw7etMIcn8TM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/miniredis v2.5.0+incompatible/go.mod h1:8HZjEj4yU0dwhYHky+DxYx+6BMjkBbe5ONFIF1MXffk= -github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= -github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/astaxie/beego v1.12.1/go.mod h1:kPBWpSANNbSdIqOc8SUL9h+1oyBMZhROeYsXQDbidWQ= -github.com/astaxie/beego v1.12.2 h1:CajUexhSX5ONWDiSCpeQBNVfTzOtPb9e9d+3vuU5FuU= -github.com/astaxie/beego v1.12.2/go.mod h1:TMcqhsbhN3UFpN+RCfysaxPAbrhox6QSS3NIAEp/uzE= +github.com/antchfx/htmlquery v1.2.4 h1:qLteofCMe/KGovBI6SQgmou2QNyedFUW+pE+BpeZ494= +github.com/antchfx/htmlquery v1.2.4/go.mod h1:2xO6iu3EVWs7R2JYqBbp8YzG50gj/ofqs5/0VZoDZLc= +github.com/antchfx/xpath v1.2.0 h1:mbwv7co+x0RwgeGAOHdrKy89GvHaGvxxBtPK0uF9Zr8= +github.com/antchfx/xpath v1.2.0/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= github.com/astaxie/beego v1.12.3 h1:SAQkdD2ePye+v8Gn1r4X6IKZM1wd28EyUOVQ3PDSOOQ= github.com/astaxie/beego v1.12.3/go.mod h1:p3qIm0Ryx7zeBHLljmd7omloyca1s4yu1a8kM1FkpIA= -github.com/beego/bee v1.12.3 h1:9gCJRAhSWtUasJsX4p3BXH9QQf7MS09PdYP3xE6iD5Q= -github.com/beego/bee v1.12.3/go.mod h1:WNn8+mpl0sbRgP3JKh6wJG3pSCpEQ8hoDMZFBb0n/Ps= github.com/beego/goyaml2 v0.0.0-20130207012346-5545475820dd/go.mod h1:1b+Y/CofkYwXMUU0OhQqGvsY2Bvgr4j6jfT699wyZKQ= github.com/beego/x2j v0.0.0-20131220205130-a0352aadc542/go.mod h1:kSeGC/p1AbBiEp5kat81+DSQrZenVBZXklMLaELspWU= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/bradfitz/gomemcache v0.0.0-20180710155616-bc664df96737/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60= github.com/casbin/casbin v1.7.0/go.mod h1:c67qKN6Oum3UF5Q1+BByfFxkwKvhwW57ITjqwtzR1KE= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cosiner/argv v0.1.0 h1:BVDiEL32lwHukgJKP87btEPenzrrHUjajs/8yzaqcXg= -github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8= -github.com/couchbase/go-couchbase v0.0.0-20181122212707-3e9b6e1258bb/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U= github.com/couchbase/go-couchbase v0.0.0-20200519150804-63f3cdb75e0d/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U= -github.com/couchbase/gomemcached v0.0.0-20181122193126-5125a94a666c/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= github.com/couchbase/gomemcached v0.0.0-20200526233749-ec430f949808/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cupcake/rdb v0.0.0-20161107195141-43ba34106c76/go.mod h1:vYwsqCOLxGiisLwp9rITslkFNpZD5rz43tf41QFkTWY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -77,42 +35,23 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/go-elasticsearch/v6 v6.8.5/go.mod h1:UwaDJsD3rWLM5rKNFzv9hgox93HoX8utj1kxD9aFUcI= github.com/elazarl/go-bindata-assetfs v1.0.0 h1:G/bYguwHIzWq9ZoyUQqrjTmJbbYn3j3CKKpKinvZLFk= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= -github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= -github.com/flosch/pongo2 v0.0.0-20200529170236-5abacdfa4915 h1:rNVrewdFbSujcoKZifC6cHJfqCTbCIR7XTLHW5TqUWU= -github.com/flosch/pongo2 v0.0.0-20200529170236-5abacdfa4915/go.mod h1:fB4mx6dzqFinCxIf3a7Mf5yLk+18Bia9mPAnuejcvDA= -github.com/flosch/pongo2 v0.0.0-20200805083417-63c99409991d h1:YTH02tThgE2P472FmZRLKUMQ3f8bq2XEDC24ZtAqNWU= -github.com/flosch/pongo2 v0.0.0-20200805083417-63c99409991d/go.mod h1:iRjKlxWTYIaf2vWBR7UdOAqduijbwbZgpsoO6oLVXRs= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/gadelkareem/delve v1.4.2-0.20200619175259-dcd01330766f h1:SXR+MNQLeyoKOHwKziU6RU8wKEaGTNhL9rkHRuKND3A= -github.com/gadelkareem/delve v1.4.2-0.20200619175259-dcd01330766f/go.mod h1:yRnaIw9CedrRtnrIhNVh1JLOz0cjEUWOEM5FaWEMOV0= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glendc/gopher-json v0.0.0-20170414221815-dc4743023d0c/go.mod h1:Gja1A+xZ9BoviGJNA2E9vFkPjjsl+CoJxSXiQM1UXtw= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-redis/redis v6.14.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -126,71 +65,23 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-dap v0.2.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e h1:JKmoR8x90Iww1ks85zJ1lfDGgIiMDuIptTOhJq+zKyg= github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= -github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= -github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= -github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= -github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= -github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= -github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= -github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= -github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/errors v0.0.0-20190930114154-d42613fe1ab9/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -199,46 +90,12 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/ledisdb/ledisdb v0.0.0-20200510135210-d35789ec47e6/go.mod h1:n931TsDuKuq+uX4v1fulaMbA/7ZLLhjc85h7chZGBCQ= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= -github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.7.0 h1:h93mCPfUSkaul3Ka/VG8uZdmW1uMHDGxzu0NWHuJmHY= -github.com/lib/pq v1.7.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.8.0 h1:9xohqzkUwzR4Ga4ivdTcawVS89YSDVxXMa3xJX3cGzg= github.com/lib/pq v1.8.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= -github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.4 h1:8KGKTcQQGm0Kv7vEbKFErAoAOFyyacLStRtQSeYtvkY= -github.com/magiconair/properties v1.8.4/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= -github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= -github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= -github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= -github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= -github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v2.0.3+incompatible h1:gXHsfypPkaMZrKbD5209QV9jbUTJKjyR5WD3HYQSd+U= github.com/mattn/go-sqlite3 v2.0.3+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= -github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= -github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.3.3 h1:SzB1nHZ2Xi+17FP0zVQBHIZqvwRN9408fJO8h+eeNA8= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -248,31 +105,19 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwd github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.0.1/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.8.1 h1:1Nf83orprkJyknT6h7zbuEGUEjcyVlCxSUGTENmNCRM= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= -github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233 h1:jmJndGFBPjNWW+MAYarU/Nl8QrQVzbw4B/AYE0LzETo= github.com/peterh/liner v1.0.1-0.20171122030339-3681c2a91233/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= -github.com/peterh/liner v1.2.0 h1:w/UPXyl5GfahFxcTOz2j9wCIHNI+pUPr2laqpojKNCg= -github.com/peterh/liner v1.2.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.0 h1:wCi7urQOGBsYcQROHqpUUX4ct84xp40t9R9JX0FuA/U= github.com/prometheus/client_golang v1.7.0/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= @@ -280,244 +125,97 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1: github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/richardlehane/mscfb v1.0.3 h1:rD8TBkYWkObWO0oLDFCbwMeZ4KoalxQy+QgniCj3nKI= +github.com/richardlehane/mscfb v1.0.3/go.mod h1:YzVpcZg9czvAuhk9T+a3avCpcFPMUWm7gK3DypaEsUk= +github.com/richardlehane/msoleps v1.0.1 h1:RfrALnSNXzmXLbGct/P2b4xkFz4e8Gmj/0Vj9M9xC1o= +github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 h1:X+yvsM2yrEktyI+b2qND5gpH8YhURn0k8OCaeRnkINo= github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644/go.mod h1:nkxAfR/5quYxwPZhyDxgasBMnRtBZd0FCEpawpjMUFg= github.com/siddontang/go v0.0.0-20170517070808-cb568a3e5cc0/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= -github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= github.com/siddontang/goredis v0.0.0-20150324035039-760763f78400/go.mod h1:DDcKzU3qCuvj/tPnimWSsZZzvk9qvkvrIL5naVBPh5s= -github.com/siddontang/ledisdb v0.0.0-20181029004158-becf5f38d373/go.mod h1:mF1DpOSOUiJRMR+FDqaqu3EBqrybQtrDDszLUZ6oxPg= github.com/siddontang/rdb v0.0.0-20150307021120-fc89ed2e418d/go.mod h1:AMEsy7v5z92TR1JKMkLLoaOQk++LVnOKL3ScbJ8GNGA= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= -github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/smartwalle/pongo2render v1.0.1 h1:rsPnDTu/+zIT5HEB5RbMjxKY5hisov26j0isZL/7YS0= -github.com/smartwalle/pongo2render v1.0.1/go.mod h1:MGnTzND7nEMz7g194kjlnw8lx/V5JJlb1hr5kDXEO0I= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY= github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= -github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.4.1 h1:asw9sl74539yqavKaglDM5hFpdJVK0Y5Dr/JOgQ89nQ= -github.com/spf13/afero v1.4.1/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= -github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= -github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= -github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= -github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.7.0 h1:xVKxvI7ouOI5I+U9s2eeiUfMaWBVoXA3AWskkrqK0VM= -github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= -github.com/spf13/viper v1.7.1 h1:pM5oEahlgWv/WnHXpgbKz7iLIxRf65tye2Ci+XFK5sk= -github.com/spf13/viper v1.7.1/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/ssdb/gossdb v0.0.0-20180723034631-88f6b59b84ec/go.mod h1:QBvMkMya+gXctz3kmljlUCu/yB3GZ6oee+dUozsezQE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= -github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v0.0.0-20160425020131-cfa635847112/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= github.com/syndtr/goleveldb v0.0.0-20181127023241-353a9fca669c/go.mod h1:Z4AUp2Km+PwemOoO/VB5AOx9XSsIItzFjoJlOSiYmn0= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v0.0.0-20171122102828-84cb69a8af83/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= github.com/wendal/errors v0.0.0-20130201093226-f66c77a7882b/go.mod h1:Q12BUT7DqIlHRmgv3RskH+UCM/4eqVMgI0EMmlSpAXc= -github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xuri/efp v0.0.0-20191019043341-b7dc4fe9aa91 h1:gp02YctZuIPTk0t7qI+wvg3VQwTPyNmSGG6ZqOsjSL8= github.com/xuri/efp v0.0.0-20191019043341-b7dc4fe9aa91/go.mod h1:uBiSUepVYMhGTfDeBKKasV4GpgBlzJ46gXUBAqV8qLk= +github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3 h1:EpI0bqf/eX9SdZDwlMmahKM+CDBgNbsXMhsN28XrM8o= +github.com/xuri/efp v0.0.0-20210322160811-ab561f5b45e3/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI= +github.com/xuri/excelize/v2 v2.5.0 h1:nDDVfX0qaDuGjAvb+5zTd0Bxxoqa1Ffv9B4kiE23PTM= +github.com/xuri/excelize/v2 v2.5.0/go.mod h1:rSu0C3papjzxQA3sdK8cU544TebhrPUoTOaGPIh0Q1A= github.com/yuin/gopher-lua v0.0.0-20171031051903-609c9cd26973/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.starlark.net v0.0.0-20190702223751-32f345186213 h1:lkYv5AKwvvduv5XWP6szk/bvvgO6aDeUujhZQXIFTes= -go.starlark.net v0.0.0-20190702223751-32f345186213/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= -go.starlark.net v0.0.0-20201113214410-e292e66a28cd h1:KKEslVKDglPoS+JDn+W1daHnbVVA4OAiyeVgOoPvzWY= -go.starlark.net v0.0.0-20201113214410-e292e66a28cd/go.mod h1:SnK8mYUdy/e3grZjI5tewfCdVSnfBHt0V8pFoqe4V2Y= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4 h1:QlVATYS7JBoZMVaf+cNjb90WD/beKVHnIxFKT4QaHVI= -golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= -golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff h1:XmKBi9R6duxOB3lfc72wyrwiOY7X2Jl1wuI+RFOyMDE= -golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.0.0-20200430140353-33d19683fad8 h1:6WW6V3x1P/jokJBpRQYUJnMHRP6isStQwCozxnU7XQw= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97 h1:/UOmuWzQfxxo9UtlXMwuQU8CMgg1eZXqTRwkSQJWKOI= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/image v0.0.0-20200430140353-33d19683fad8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= -golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb h1:fqpd0EBDzlHRCjiphRR5Zo/RSWWQlWv34418dnEixWk= +golang.org/x/image v0.0.0-20210220032944-ac19c3e999fb/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f h1:QBjCr1Fz5kw158VqdE9JfI9cJnl/ymnJWAdMuinqL7Y= +golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985 h1:4CSI6oo7cOjJKajidEljs9h+uP0rRZBPPPhcCbj5mw8= +golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200819091447-39769834ee22 h1:YUxhQGxYV280Da2a0XZiHblyJZN6NuXS1f4dahsm0SM= -golang.org/x/sys v0.0.0-20200819091447-39769834ee22/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48 h1:AYCWBZhgIw6XobZ5CibNJr0Rc4ZofGGKvWa1vcx2IGk= -golang.org/x/sys v0.0.0-20201116194326-cc9327a14d48/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7 h1:s330+6z/Ko3J0o6rvOcwXe5nzs7UT9tLKHoOXYn6uE0= -golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= -golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200117065230-39095c1d176c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -529,23 +227,14 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= -gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= -gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -553,9 +242,5 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/cve-vulner-manager/main.go b/cve-vulner-manager/main.go index 2606737..eb6d1a5 100644 --- a/cve-vulner-manager/main.go +++ b/cve-vulner-manager/main.go @@ -2,6 +2,7 @@ package main import ( "cvevulner/common" + cve_timed_task "cvevulner/cve-timed-task" "cvevulner/models" _ "cvevulner/models" _ "cvevulner/routers" @@ -30,6 +31,11 @@ func main() { task.StopTask() return } + + err := cve_timed_task.StartCveTimedTask() + if err != nil { + return + } // single run task.StartTask() defer task.StopTask() diff --git a/cve-vulner-manager/models/initdb.go b/cve-vulner-manager/models/initdb.go index 05e4c9a..b5a071f 100644 --- a/cve-vulner-manager/models/initdb.go +++ b/cve-vulner-manager/models/initdb.go @@ -23,9 +23,9 @@ func Initdb() bool { dbuser := BConfig.String("mysql::dbuser") dbname := BConfig.String("mysql::dbname") dbpwd := BConfig.String("mysql::dbpwd") - key := beego.AppConfig.String("key") - key1 := []byte(key) - bytes, _ := common.DePwdCode(dbpwd, key1) + //key := beego.AppConfig.String("key") + //key1 := []byte(key) + //bytes, _ := common.DePwdCode(dbpwd, key1) maxidle, lerr := BConfig.Int("mysql::maxidle") if lerr != nil { maxidle = 30 @@ -35,7 +35,8 @@ func Initdb() bool { if lerr != nil { maxconn = 3000 } - dns := dbuser + ":" + string(bytes) + "@tcp(" + dbhost + ":" + dbport + ")/" + dbname + "?charset=utf8" + //dns := dbuser + ":" + string(bytes) + "@tcp(" + dbhost + ":" + dbport + ")/" + dbname + "?charset=utf8" + dns := dbuser + ":" + dbpwd + "@tcp(" + dbhost + ":" + dbport + ")/" + dbname + "?charset=utf8" errx := orm.RegisterDriver("mysql", orm.DRMySQL) if errx != nil { logs.Error("RegisterDriver, orm err: ", errx) -- Gitee From f7ac51b6cd068da0e69dff28e13978ec3c889aa3 Mon Sep 17 00:00:00 2001 From: "775517776@qq.com" <775517776@qq.com> Date: Wed, 2 Mar 2022 14:42:10 +0800 Subject: [PATCH 2/8] =?UTF-8?q?=E2=80=98add=20time=20task=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cve-vulner-manager/conf/app.conf | 2 + .../controller/task_controller.go | 55 ++++++++++--------- .../controller/timer_task_controller.go | 10 ++-- .../downloadfiletask/download_file_task.go | 19 +++---- .../cve-timed-task/gitwebtask/gene_group.go | 11 ++-- .../cve-timed-task/tabletask/common.go | 6 +- .../cve-timed-task/tabletask/crawltask.go | 5 -- .../cve-timed-task/tabletask/gauss_yaml.go | 3 +- .../cve-timed-task/tabletask/mapping_task.go | 31 ++++++----- .../tabletask/mindspore_yaml.go | 3 +- .../tabletask/package_committer_task.go | 1 - .../cve-timed-task/tabletask/repeat_task.go | 6 +- .../tabletask/supplement_cve.go | 4 +- .../cve-timed-task/util/crawl.go | 5 -- 14 files changed, 76 insertions(+), 85 deletions(-) diff --git a/cve-vulner-manager/conf/app.conf b/cve-vulner-manager/conf/app.conf index 06247fc..93a2b4e 100644 --- a/cve-vulner-manager/conf/app.conf +++ b/cve-vulner-manager/conf/app.conf @@ -35,6 +35,8 @@ rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEul #communityName = "src-openeuler:1,opengauss:2,mindspore:3" communityName = "cve-test:1,cve-gauss:2,cve-mindspore:3,open-loo-keng:4" +[timedtask] +packageDir = "cve-timed-task/" ; [mysql] ; dbhost = "${DB_URI||127.0.0.1}" diff --git a/cve-vulner-manager/cve-timed-task/controller/task_controller.go b/cve-vulner-manager/cve-timed-task/controller/task_controller.go index 43e56ee..fe1371e 100644 --- a/cve-vulner-manager/cve-timed-task/controller/task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/task_controller.go @@ -6,35 +6,36 @@ import ( "cvevulner/cve-timed-task/gitwebtask" "cvevulner/cve-timed-task/tabletask" "cvevulner/cve-timed-task/util" + "github.com/astaxie/beego" "github.com/astaxie/beego/logs" "time" ) // GwController Grab warehouse yaml data func GwController() { - logs.Info("Start of crawling organization, warehouse, owners correspondence relationship timing task") + logs.Info("START TASK:", "Start of crawling organization, warehouse, owners correspondence relationship timing task") gitwebtask.Handle() //TODO - logs.Info("Grab warehouse yaml data task completed") + logs.Info("END TASK:", "Grab warehouse yaml data task completed") } // RunTableTask Parse the artificial CVE table, crawl the CVE official //website data, and store it in the database func RunTableTask() { - logs.Info("Analyze the manual data table, crawl the CVE official website data, and start the timing task") + logs.Info("START TASK:", "Analyze the manual data table, crawl the CVE official website data, and start the timing task") tabletask.AddErrorDetails() tabletask.HandleDate() tabletask.RepeatTask() tabletask.GetPublishedDateTask() - logs.Info("Analyze the manual data table, crawl the CVE official website data task completed") + logs.Info("END TASK:", "Analyze the manual data table, crawl the CVE official website data task completed") } // ToExcelController Query the data whose cve_status status is 3 or 4 in the // database to generate excel Modify the status of cve_status to 3 or 4 to 5, // indicating that it has been added to excel to be processed func ToExcelController() { - logs.Info("Export data to generate excel, send error data feedback email task start") - filePath := "problemexcels/" + logs.Info("START TASK:", "Export data to generate excel, send error data feedback email task start") + filePath := beego.AppConfig.String("timedtask::packageDir") + "problemexcels/" err := util.MakeDir(filePath) if err != nil { logs.Error("util.makeDir error", err) @@ -44,28 +45,28 @@ func ToExcelController() { tabletask.GenerateTable(filePath + "error_data_" + createTime + ".xls") emailtask.SendingTypeTwo(filePath) - logs.Info("To excel and send email task completed") + logs.Info("END TASK:", "To excel and send email task completed") } // RunMappingTask Parse the artificial CVE table, crawl the CVE //official website data, and store it in the database func RunMappingTask() { - logs.Info("The timing task of parsing the package name mapping table starts") + logs.Info("START TASK:", "The timing task of parsing the package name mapping table starts") tabletask.ToMysql() - logs.Info("The timing task of parsing the package name mapping table completed") + logs.Info("END TASK:", "The timing task of parsing the package name mapping table completed") } // RunDelFileTask Delete files older than one month func RunDelFileTask() { - logs.Info("Clean up the folder timed task starts") - deletetask.DelDir("oldexcels/") - logs.Info("Clean up the folder task completed") + logs.Info("START TASK:", "Clean up the folder timed task starts") + deletetask.DelDir(beego.AppConfig.String("timedtask::packageDir") + "oldexcels/") + logs.Info("END TASK:", "Clean up the folder task completed") } // RunIssueRecordTask issues record email task func RunIssueRecordTask() { - logs.Info("issues record email task start") - filePath := "export_excels/" + logs.Info("START TASK:", "issues record email task start") + filePath := beego.AppConfig.String("timedtask::packageDir") + "export_excels/" err := util.MakeDir(filePath) if err != nil { logs.Error("util.MakeDir error:", err, filePath) @@ -74,48 +75,48 @@ func RunIssueRecordTask() { tabletask.GenerateExcels("1", filePath+"create_issues.xls") tabletask.GenerateExcels("2", filePath+"unctreated_issues.xls") emailtask.Sending(filePath) - logs.Info("issues record email task complete") + logs.Info("END TASK:", "issues record email task complete") } // RunWhitelistTask the task of parsing package whitelist func RunWhitelistTask() { - logs.Info("the task of parsing package whitelist start") + logs.Info("START TASK:", "the task of parsing package whitelist start") tabletask.ImportDataTypeTwo() - logs.Info("the task of parsing package whitelist complete") + logs.Info("END TASK:", "the task of parsing package whitelist complete") } // RunPackageCommitterTask the task of parsing package_committer excels func RunPackageCommitterTask() { - logs.Info("the task of parsing package_committer_excels start") + logs.Info("START TASK:", "the task of parsing package_committer_excels start") tabletask.ImportData() - logs.Info("the task of parsing package_committer_excels complete") + logs.Info("END TASK:", "the task of parsing package_committer_excels complete") } // SupplementCveTask Complete the template information of the issue with the data on the CVE official website func SupplementCveTask() { - logs.Info("Supplemental cve information task starts") + logs.Info("START TASK:", "Supplemental cve information task starts") tabletask.SupplementCve() - logs.Info("Supplement cve information task is over") + logs.Info("END TASK:", "Supplement cve information task is over") } // LongSupplementCveTask Complete the template information of the //issue with the data on the CVE official website func LongSupplementCveTask() { - logs.Info("long Supplemental cve information task starts") + logs.Info("START TASK:", "long Supplemental cve information task starts") tabletask.LongSupplementCve() - logs.Info("long Supplement cve information task is over") + logs.Info("END TASK:", "long Supplement cve information task is over") } // ParseOpenGaussYamlTask Download the yaml file first, then parse the yaml file of opengauss func ParseOpenGaussYamlTask() { - logs.Info("the task of Process openGauss yaml data start") + logs.Info("START TASK:", "the task of Process openGauss yaml data start") tabletask.ProcGaussYaml() - logs.Info("the task of Process openGauss yaml data complete") + logs.Info("END TASK:", "the task of Process openGauss yaml data complete") } // ParseMindsporeYamlTask Download the yaml file first, then parse the yaml file of mindspore func ParseMindsporeYamlTask() { - logs.Info("the task of Process mindspore yaml data start") + logs.Info("START TASK:", "the task of Process mindspore yaml data start") tabletask.ProcMindSporeYaml() - logs.Info("the task of Process mindspore yaml data complete") + logs.Info("END TASK:", "the task of Process mindspore yaml data complete") } diff --git a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go index 836aca4..39f204d 100644 --- a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go @@ -14,7 +14,7 @@ func StartTimerTask() error { logs.Error("add GwController task error:", err.Error()) return err } - logs.Info("Add ToExcelController task success, ID is:", IdGwController) + logs.Info("Add GwController task success, ID is:", IdGwController) IdRunTableTask, err := c.AddFunc("@every 40m", RunTableTask) if err != nil { @@ -77,21 +77,21 @@ func StartTimerTask() error { logs.Error("add SupplementCveTask task error:", err.Error()) return err } - logs.Info("Add IdParseOpenGaussYamlTask task success, ID is:", IdSupplementCveTask) + logs.Info("Add SupplementCveTask task success, ID is:", IdSupplementCveTask) IdLongSupplementCveTask, err := c.AddFunc("30 1 * * 0-6", LongSupplementCveTask) if err != nil { logs.Error("add LongSupplementCveTask task error:", err.Error()) return err } - logs.Info("Add IdLongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) + logs.Info("Add LongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) //Parse opengauss yaml file IdParseOpenGaussYamlTask, err := c.AddFunc("30 2 * * 0-6", ParseOpenGaussYamlTask) if err != nil { logs.Error("add ParseOpenGaussYamlTask task error:", err.Error()) return err } - logs.Info("Add IdParseOpenGaussYamlTask task success, ID is:", IdParseOpenGaussYamlTask) + logs.Info("Add ParseOpenGaussYamlTask task success, ID is:", IdParseOpenGaussYamlTask) //Parse the yaml file of mindspore IdParseMindsporeYamlTask, err := c.AddFunc("30 3 * * 0-6", ParseMindsporeYamlTask) @@ -99,7 +99,7 @@ func StartTimerTask() error { logs.Error("add ParseMindsporeYamlTask task error:", err.Error()) return err } - logs.Info("Add IdParseOpenGaussYamlTask task success, ID is:", IdParseMindsporeYamlTask) + logs.Info("Add ParseMindsporeYamlTask task success, ID is:", IdParseMindsporeYamlTask) c.Start() return nil } diff --git a/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go b/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go index 256a4a1..b58af19 100644 --- a/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go +++ b/cve-vulner-manager/cve-timed-task/downloadfiletask/download_file_task.go @@ -4,6 +4,7 @@ import ( "cvevulner/cve-timed-task/util" "fmt" "github.com/antchfx/htmlquery" + "github.com/astaxie/beego" "github.com/astaxie/beego/logs" "io" "net/http" @@ -12,7 +13,7 @@ import ( ) func HandleOne() (string, error) { - filePath := "newexcels/" + filePath := beego.AppConfig.String("timedtask::packageDir") + "newexcels/" err := util.MakeDir(filePath) if err != nil { logs.Error("util.makeDir error", err) @@ -22,12 +23,13 @@ func HandleOne() (string, error) { url := "https://gitee.com/openeuler/cve-manager/tree/master/cve-vulner-manager/cve-py/newexcels" html, err := util.UrlToHTML(url) if err != nil { + logs.Error("util.UrlToHTML error: ", err, url) return "", err } for i := 3; ; i += 2 { names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) if err != nil { - logs.Error("xpath match error:", err) + logs.Error("htmlquery.QueryAll error:", err) return "", err } if len(names) == 0 { @@ -71,7 +73,7 @@ func HandleOne() (string, error) { } func HandleTwo() (string, error) { - filePath := "mappingexcels/" + filePath := beego.AppConfig.String("timedtask::packageDir") + "mappingexcels/" err := util.MakeDir(filePath) if err != nil { logs.Error("util.makeDir error", err) @@ -129,7 +131,7 @@ func HandleTwo() (string, error) { } func HandleThree() (string, error) { - filePath := "import_excels/" + filePath := beego.AppConfig.String("timedtask::packageDir") + "import_excels/" err := util.MakeDir(filePath) if err != nil { logs.Error("util.makeDir error", err) @@ -191,7 +193,7 @@ func HandleThree() (string, error) { } func HandleFour() (string, error) { - filePath := "package_committer_excels/" + filePath := beego.AppConfig.String("timedtask::packageDir") + "package_committer_excels/" err := util.MakeDir(filePath) if err != nil { logs.Error("util.makeDir error", err) @@ -233,7 +235,6 @@ func HandleFour() (string, error) { if err != nil { logs.Error("create file error:", saveFile) _ = resp.Body.Close() - _ = saveFile.Close() continue } _, err = io.Copy(saveFile, resp.Body) @@ -254,7 +255,7 @@ func HandleFour() (string, error) { // DownloadSpecErrorExcels download spec_error_excels func DownloadSpecErrorExcels() (string, error) { - filePath := "spec_error_excels/" + filePath := beego.AppConfig.String("timedtask::packageDir") + "spec_error_excels/" err := util.MakeDir(filePath) if err != nil { logs.Error("util.makeDir error", err) @@ -270,7 +271,7 @@ func DownloadSpecErrorExcels() (string, error) { for i := 3; ; i += 2 { names, err := htmlquery.QueryAll(html, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@title", i)) if err != nil { - logs.Error("xpath match error:", err) + logs.Error("htmlquery.QueryAll(:", err) return "", err } if len(names) == 0 { @@ -282,7 +283,6 @@ func DownloadSpecErrorExcels() (string, error) { resp, err := http.Get(downloadUrl) if err != nil { logs.Error("GET", url, "error:", err.Error()) - _ = resp.Body.Close() continue } @@ -296,7 +296,6 @@ func DownloadSpecErrorExcels() (string, error) { if err != nil { logs.Error("create file error:", saveFile) _ = resp.Body.Close() - _ = saveFile.Close() continue } _, err = io.Copy(saveFile, resp.Body) diff --git a/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go b/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go index 18791c0..e817f41 100644 --- a/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go +++ b/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go @@ -69,15 +69,17 @@ func Handle() { logs.Error(" Rollback transaction error:", tranErr.Error()) } } - } - tranErr = db_models.ORM.Commit() - if tranErr != nil { - logs.Error(" Commit transaction error:", tranErr.Error()) + } else { + tranErr = db_models.ORM.Commit() + if tranErr != nil { + logs.Error(" Commit transaction error:", tranErr.Error()) + } } sigPageNodes, err := htmlquery.QueryAll(doc, fmt.Sprintf("//div[@id='tree-slider']/div[%d]/div[1]/a/@href", i)) if err != nil { logs.Warn("htmlquery.QueryAll error:", err) + continue } sigPage := htmlquery.SelectAttr(sigPageNodes[0], "href") @@ -90,7 +92,6 @@ func Handle() { //Get the name, homepage and etherpad of all sigs sigsList = append(sigsList, s) } - fmt.Println(sigsList) for _, s := range sigsList { gitRepoGroup, err := db_models.GetOneGitRepoGroups(s.sigName) if err != nil { diff --git a/cve-vulner-manager/cve-timed-task/tabletask/common.go b/cve-vulner-manager/cve-timed-task/tabletask/common.go index 7bca973..c8945cd 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/common.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/common.go @@ -50,13 +50,13 @@ func DeleteYamlData() { continue } } else { - if errg != nil { + if errg != nil && !errors.Is(orm.ErrNoRows, errg) { logs.Error("db_models.SelectGaussYamlOriginData:", errg, v.PackageName, v.Version) } - if errm != nil { + if errm != nil && errors.Is(orm.ErrNoRows, errm) { logs.Error("db_models.SelectMindSporeYamlOriginData", errm, v.PackageName, v.Version) } - if erro != nil { + if erro != nil && errors.Is(orm.ErrNoRows, erro) { logs.Error("db_models.SelectOpenLookengYamlOriginData", erro, v.PackageName, v.Version) } } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go index e50ccd7..da06e58 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go @@ -31,7 +31,6 @@ type XpathList struct { // Crawling Grab cve specific information func Crawling(url string) (XpathList, error) { - logs.Info("now crawling :", url) xpathList := XpathList{} if url == "" || !strings.Contains(url, "http") { @@ -56,8 +55,6 @@ func Crawling(url string) (XpathList, error) { } else { xpathList.CveDesc = cveDesc } - } else { - logs.Info("xpath no match") } } //xpathList.RepairTime @@ -79,8 +76,6 @@ func Crawling(url string) (XpathList, error) { } } } - } else { - logs.Info("xpath no match") } } xpathList.ScoreType = "v3.0" diff --git a/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go index 3def27b..9b095e2 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go @@ -5,6 +5,7 @@ import ( "cvevulner/cve-timed-task/util" "cvevulner/models" "errors" + "github.com/astaxie/beego" "github.com/astaxie/beego/logs" "github.com/astaxie/beego/orm" "io" @@ -15,7 +16,7 @@ import ( // DownloadGuessYaml Download the yaml file of openGauss on Gitee func DownloadGuessYaml() (string, error) { - filePath := "gauss_yaml/" + filePath := beego.AppConfig.String("timedtask::packageDir") + "gauss_yaml/" err := util.MakeDir(filePath) if err != nil { return "", err diff --git a/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go b/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go index f2bcf91..a3dd6a6 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go @@ -3,9 +3,11 @@ package tabletask import ( "crypto/sha1" "cvevulner/cve-timed-task/db_models" + "cvevulner/cve-timed-task/downloadfiletask" "cvevulner/cve-timed-task/util" "cvevulner/models" "encoding/hex" + "github.com/astaxie/beego" "github.com/astaxie/beego/logs" "github.com/astaxie/beego/orm" "github.com/pkg/errors" @@ -17,12 +19,11 @@ import ( // ToMysql Data stored in the database func ToMysql() { - //filePath, err := downloadfiletask.HandleTwo() - //if err != nil { - // logs.Error("downloadfiletask.HandleTwo error:", err) - // return - //} - filePath := "mappingexcels/" + filePath, err := downloadfiletask.HandleTwo() + if err != nil { + logs.Error("downloadfiletask.HandleTwo error:", err) + return + } files, err := os.ReadDir(filePath) if err != nil { logs.Error(" os.ReadDir error: ", err, filePath) @@ -38,7 +39,6 @@ func ToMysql() { } f, err := os.Open(filePath + fileName) if err != nil { - _ = f.Close() continue } @@ -76,21 +76,21 @@ func ToMysql() { logs.Error("Rollback transaction error:", tranErr.Error()) } - _, err = os.Stat("oldexcels/") + _, err = os.Stat(beego.AppConfig.String("timedtask::packageDir") + "oldexcels/") if err != nil { logs.Error("os.Stat error:", err.Error()) if os.IsNotExist(err) { logs.Info("And there is no such directory", filePath, "start to create") - err = os.Mkdir("oldexcels/", os.ModePerm) + err = os.Mkdir(beego.AppConfig.String("timedtask::packageDir")+"oldexcels/", os.ModePerm) if err != nil { logs.Error("os.Mkdir error:", filePath, err.Error()) } logs.Info("create directory success:", filePath) } } - err = os.Rename(filePath+fileName, "oldexcels/"+fileName) + err = os.Rename(filePath+fileName, beego.AppConfig.String("timedtask::packageDir")+"oldexcels/"+fileName) if err != nil { - logs.Error("os.Rename error:", filePath+fileName, "to", "oldexcels/"+fileName, err.Error()) + logs.Error(err.Error()) } continue } @@ -190,21 +190,22 @@ func ToMysql() { continue } } - _, err = os.Stat("oldexcels/") + _, err = os.Stat(beego.AppConfig.String("timedtask::packageDir") + "oldexcels/") if err != nil { logs.Error("os.Stat error:", err.Error()) if os.IsNotExist(err) { logs.Info("And there is no such directory", filePath, "start to create") - err = os.Mkdir("oldexcels/", os.ModePerm) + err = os.Mkdir(beego.AppConfig.String("timedtask::packageDir")+"oldexcels/", os.ModePerm) if err != nil { logs.Error("os.Mkdir error:", filePath, err.Error()) } logs.Info("create directory success:", filePath) } } - err = os.Rename(filePath+fileName, "oldexcels/"+fileName) + err = os.Rename(filePath+fileName, beego.AppConfig.String("timedtask::packageDir")+"oldexcels/"+fileName) if err != nil { - logs.Error("os.Rename error:", filePath+fileName, "to", "oldexcels/"+fileName, err.Error()) + logs.Error(err.Error()) } + continue } } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go index e9f611d..4bbe08b 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go @@ -5,6 +5,7 @@ import ( "cvevulner/cve-timed-task/util" "cvevulner/models" "errors" + "github.com/astaxie/beego" "github.com/astaxie/beego/logs" "github.com/astaxie/beego/orm" "io" @@ -15,7 +16,7 @@ import ( // DownloadGuessYaml Download the yaml file of openGauss on Gitee func DownloadMindSporeYaml() (string, error) { - filePath := "mindspore_yaml/" + filePath := beego.AppConfig.String("timedtask::packageDir") + "mindspore_yaml/" err := util.MakeDir(filePath) if err != nil { return "", err diff --git a/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go b/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go index 0c9d05a..bc24825 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go @@ -37,7 +37,6 @@ func ImportData() { } f, err := os.Open(filePath + fileName) if err != nil { - _ = f.Close() continue } hash := sha1.New() diff --git a/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go b/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go index 8fec61a..35ce2f2 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go @@ -22,7 +22,6 @@ func RepeatTask() { if rowsAffected == 0 { return } - fmt.Println(rowsAffected) now := time.Now().Format("2006-01-02 15:04:05") for _, v := range originExcel { url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", v.CveNum) @@ -76,7 +75,6 @@ func GetPublishedDateTask() { if rowsAffected == 0 { return } - fmt.Println(rowsAffected) for _, v := range vulnCenter { url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", v.CveNum) crawlList, err := Crawling(url) @@ -85,9 +83,9 @@ func GetPublishedDateTask() { continue } if crawlList.RepairTime != "" { - fmt.Println("upadd") - err := db_models.UpdateCveVulnCenter(crawlList.RepairTime, strconv.FormatInt(v.CveId, 10)) + err = db_models.UpdateCveVulnCenter(crawlList.RepairTime, strconv.FormatInt(v.CveId, 10)) if err != nil { + logs.Error("db_models.UpdateCveVulnCenter error:", err) continue } } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go index 1e64a89..e436b0c 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go @@ -128,9 +128,8 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { cveStatusx = 1 } listx, err := Crawling(url) - fmt.Println(listx) if err != nil { - logs.Error("crawling data error:", err) + logs.Error("crawling data error:", err, url) return err } @@ -191,7 +190,6 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { } } else { if score.NVDScore == 0 { - fmt.Println("up2") err := db_models.UpdateCveScore(models.Score{ NVDScore: nvdScore, NvectorVule: listx.VectorValue, diff --git a/cve-vulner-manager/cve-timed-task/util/crawl.go b/cve-vulner-manager/cve-timed-task/util/crawl.go index 7f146b7..71af4ef 100644 --- a/cve-vulner-manager/cve-timed-task/util/crawl.go +++ b/cve-vulner-manager/cve-timed-task/util/crawl.go @@ -1,7 +1,6 @@ package util import ( - "github.com/astaxie/beego/logs" "github.com/pkg/errors" "golang.org/x/net/html" "golang.org/x/net/html/charset" @@ -12,21 +11,17 @@ func UrlToHTML(url string) (*html.Node, error) { resp, err := http.Get(url) defer resp.Body.Close() if err != nil { - logs.Error("GET", url, "error:", err.Error()) return nil, err } if resp.StatusCode != http.StatusOK { - logs.Error("response status code is:", resp.Status) return nil, errors.New(resp.Status) } r, err := charset.NewReader(resp.Body, resp.Header.Get("Content-Type")) if err != nil { - logs.Error("charset.NewReader error:", err.Error()) return nil, err } doc, err := html.Parse(r) if err != nil { - logs.Error("html.Parse error:", err.Error()) return nil, err } return doc, nil -- Gitee From 36a116d58799f31f810d9c62c23975cc9256b85f Mon Sep 17 00:00:00 2001 From: "775517776@qq.com" <775517776@qq.com> Date: Thu, 3 Mar 2022 09:59:30 +0800 Subject: [PATCH 3/8] =?UTF-8?q?=E2=80=98=E4=BF=AE=E6=94=B9bug=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cve-vulner-manager/cve-timed-task/util/makrdir.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cve-vulner-manager/cve-timed-task/util/makrdir.go b/cve-vulner-manager/cve-timed-task/util/makrdir.go index c771d22..524c9ec 100644 --- a/cve-vulner-manager/cve-timed-task/util/makrdir.go +++ b/cve-vulner-manager/cve-timed-task/util/makrdir.go @@ -13,9 +13,9 @@ func MakeDir(filePath string) error { logs.Error("os.Stat error:", err.Error()) if os.IsNotExist(err) { logs.Info("And there is no such directory", filePath, "start to create") - err = os.Mkdir(filePath, os.ModePerm) + err = os.MkdirAll(filePath, os.ModePerm) if err != nil { - logs.Error("os.Mkdir error:", filePath, err.Error()) + logs.Error("os.MkdirAll error:", filePath, err.Error()) return err } logs.Info("create directory success:", filePath) -- Gitee From bfa8e4b2976615fc439d1e50571b6b3576dd2bb7 Mon Sep 17 00:00:00 2001 From: "775517776@qq.com" <775517776@qq.com> Date: Thu, 3 Mar 2022 16:59:18 +0800 Subject: [PATCH 4/8] =?UTF-8?q?=E2=80=98=E4=BF=AE=E6=94=B9bug=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cve-vulner-manager/conf/app.conf | 16 ++++++++++++ .../controller/timer_task_controller.go | 25 ++++++++++--------- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cve-vulner-manager/conf/app.conf b/cve-vulner-manager/conf/app.conf index 93a2b4e..f70de9a 100644 --- a/cve-vulner-manager/conf/app.conf +++ b/cve-vulner-manager/conf/app.conf @@ -37,6 +37,22 @@ communityName = "cve-test:1,cve-gauss:2,cve-mindspore:3,open-loo-keng:4" [timedtask] packageDir = "cve-timed-task/" +GwController = "2 0 * * 0-6" +RunTableTask = "@every 40m" +ToExcelController = "2 0 * * 0-6" +RunMappingTask = "30 7 * * 0-6" +RunDelFileTask = "30 9 * * 0-6" +RunIssueRecordTask = "30 7 * * 0-6" +RunWhitelistTask = "@every 2h" +RunPackageCommitterTask = "@every 1h" +SupplementCveTask = "@every 27m" +LongSupplementCveTask = "30 1 * * 0-6" +ParseOpenGaussYamlTask = "30 2 * * 0-6" +ParseMindsporeYamlTask = "30 3 * * 0-6" + + + + ; [mysql] ; dbhost = "${DB_URI||127.0.0.1}" diff --git a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go index 39f204d..b770868 100644 --- a/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go +++ b/cve-vulner-manager/cve-timed-task/controller/timer_task_controller.go @@ -1,6 +1,7 @@ package controller import ( + "github.com/astaxie/beego" "github.com/astaxie/beego/logs" "github.com/robfig/cron/v3" ) @@ -9,14 +10,14 @@ func StartTimerTask() error { c := cron.New() //Warehouse, organization, and owner correspondence timing task - IdGwController, err := c.AddFunc("2 0 * * 0-6", GwController) + IdGwController, err := c.AddFunc(beego.AppConfig.String("timedtask::GwController"), GwController) if err != nil { logs.Error("add GwController task error:", err.Error()) return err } logs.Info("Add GwController task success, ID is:", IdGwController) - IdRunTableTask, err := c.AddFunc("@every 40m", RunTableTask) + IdRunTableTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunTableTask"), RunTableTask) if err != nil { logs.Error("add RunTableTask task error:", err.Error()) return err @@ -24,7 +25,7 @@ func StartTimerTask() error { logs.Info("Add RunTableTask task success, ID is:", IdRunTableTask) //Error data export and email sending timing task - IdToExcelController, err := c.AddFunc("2 0 * * 0-6", ToExcelController) + IdToExcelController, err := c.AddFunc(beego.AppConfig.String("timedtask::ToExcelController"), ToExcelController) if err != nil { logs.Error("add ToExcelController task error:", err.Error()) return err @@ -32,7 +33,7 @@ func StartTimerTask() error { logs.Info("Add ToExcelController task success, ID is:", IdToExcelController) //Package name correspondence relationship timing task - IdRunMappingTask, err := c.AddFunc("30 7 * * 0-6", RunMappingTask) + IdRunMappingTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunMappingTask"), RunMappingTask) if err != nil { logs.Error("add RunMappingTask task error:", err.Error()) return err @@ -40,7 +41,7 @@ func StartTimerTask() error { logs.Info("Add RunMappingTask task success, ID is:", IdRunMappingTask) //Delete ,iles timed tasks that are more than one month old - IdRunDelFileTask, err := c.AddFunc("30 9 * * 0-6", RunDelFileTask) + IdRunDelFileTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunDelFileTask"), RunDelFileTask) if err != nil { logs.Error("add RunDelFileTask task error:", err.Error()) return err @@ -48,7 +49,7 @@ func StartTimerTask() error { logs.Info("Add RunDelFileTask task success, ID is:", IdRunDelFileTask) //issue record email feedback - IdRunIssueRecordTask, err := c.AddFunc("30 7 * * 0-6", RunIssueRecordTask) + IdRunIssueRecordTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunIssueRecordTask"), RunIssueRecordTask) if err != nil { logs.Error("add RunIssueRecordTask task error:", err.Error()) return err @@ -56,7 +57,7 @@ func StartTimerTask() error { logs.Info("Add RunIssueRecordTask task success, ID is:", IdRunIssueRecordTask) //analysis the table package whitelist - IdRunWhitelistTask, err := c.AddFunc("@every 2h", RunWhitelistTask) + IdRunWhitelistTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunWhitelistTask"), RunWhitelistTask) if err != nil { logs.Error("add RunWhitelistTask task error:", err.Error()) return err @@ -64,7 +65,7 @@ func StartTimerTask() error { logs.Info("Add RunWhitelistTask task success, ID is:", IdRunWhitelistTask) //parsing the table package_committer_excels - IdRunPackageCommitterTask, err := c.AddFunc("@every 1h", RunPackageCommitterTask) + IdRunPackageCommitterTask, err := c.AddFunc(beego.AppConfig.String("timedtask::RunPackageCommitterTask"), RunPackageCommitterTask) if err != nil { logs.Error("add RunPackageCommitterTask task error:", err.Error()) return err @@ -72,21 +73,21 @@ func StartTimerTask() error { logs.Info("Add RunPackageCommitterTask task success, ID is:", IdRunPackageCommitterTask) //Complete the template information of the issue with the data on the CVE official website - IdSupplementCveTask, err := c.AddFunc("@every 27m", SupplementCveTask) + IdSupplementCveTask, err := c.AddFunc(beego.AppConfig.String("timedtask::SupplementCveTask"), SupplementCveTask) if err != nil { logs.Error("add SupplementCveTask task error:", err.Error()) return err } logs.Info("Add SupplementCveTask task success, ID is:", IdSupplementCveTask) - IdLongSupplementCveTask, err := c.AddFunc("30 1 * * 0-6", LongSupplementCveTask) + IdLongSupplementCveTask, err := c.AddFunc(beego.AppConfig.String("timedtask::LongSupplementCveTask"), LongSupplementCveTask) if err != nil { logs.Error("add LongSupplementCveTask task error:", err.Error()) return err } logs.Info("Add LongSupplementCveTask task success, ID is:", IdLongSupplementCveTask) //Parse opengauss yaml file - IdParseOpenGaussYamlTask, err := c.AddFunc("30 2 * * 0-6", ParseOpenGaussYamlTask) + IdParseOpenGaussYamlTask, err := c.AddFunc(beego.AppConfig.String("timedtask::ParseOpenGaussYamlTask"), ParseOpenGaussYamlTask) if err != nil { logs.Error("add ParseOpenGaussYamlTask task error:", err.Error()) return err @@ -94,7 +95,7 @@ func StartTimerTask() error { logs.Info("Add ParseOpenGaussYamlTask task success, ID is:", IdParseOpenGaussYamlTask) //Parse the yaml file of mindspore - IdParseMindsporeYamlTask, err := c.AddFunc("30 3 * * 0-6", ParseMindsporeYamlTask) + IdParseMindsporeYamlTask, err := c.AddFunc(beego.AppConfig.String("timedtask::ParseMindsporeYamlTask"), ParseMindsporeYamlTask) if err != nil { logs.Error("add ParseMindsporeYamlTask task error:", err.Error()) return err -- Gitee From 1864ac8e1ed6eea2ee6e990006f3aff0e5caa9b8 Mon Sep 17 00:00:00 2001 From: "775517776@qq.com" <775517776@qq.com> Date: Fri, 4 Mar 2022 10:12:59 +0800 Subject: [PATCH 5/8] =?UTF-8?q?=E2=80=98=E4=BF=AE=E6=94=B9bug=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cve-vulner-manager/conf/product_app.conf | 33 +++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index 1d3044e..fff0ba3 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -35,11 +35,38 @@ rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEul communityName = "src-openeuler:1,opengauss:2,mindspore:3,openlookeng:4" + +[timedtask] +packageDir = "cve-timed-task/" +GwController = "2 0 * * 0-6" +RunTableTask = "@every 40m" +ToExcelController = "2 0 * * 0-6" +RunMappingTask = "30 7 * * 0-6" +RunDelFileTask = "30 9 * * 0-6" +RunIssueRecordTask = "30 7 * * 0-6" +RunWhitelistTask = "@every 2h" +RunPackageCommitterTask = "@every 1h" +SupplementCveTask = "@every 27m" +LongSupplementCveTask = "30 1 * * 0-6" +ParseOpenGaussYamlTask = "30 2 * * 0-6" +ParseMindsporeYamlTask = "30 3 * * 0-6" + +; [mysql] +; dbhost = "${DB_URI||***}" +; dbport = 3306 +; dbuser = "${DB_USER||cve}" +; dbpwd = "${DB_PWD||***}" +; dbname = cvevulner +; dbprefix = cve_ +; maxidle = 30 +; maxconn = 3000 + + [mysql] -dbhost = "${DB_URI||***}" +dbhost = "127.0.0.1" dbport = 3306 -dbuser = "${DB_USER||cve}" -dbpwd = "${DB_PWD||***}" +dbuser = "root" +dbpwd = "root" dbname = cvevulner dbprefix = cve_ maxidle = 30 -- Gitee From ce8ac343d295f876a08f96838e2ed4501c38fd43 Mon Sep 17 00:00:00 2001 From: "775517776@qq.com" <775517776@qq.com> Date: Mon, 7 Mar 2022 14:55:34 +0800 Subject: [PATCH 6/8] =?UTF-8?q?=E2=80=98=E4=BF=AE=E6=94=B9bug=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cve-vulner-manager/conf/product_app.conf | 2 +- .../cve-timed-task/db_models/db_operations.go | 134 +++++++++--------- .../cve-timed-task/db_models/init_db.go | 13 -- .../emailtask/issue_record_email.go | 4 +- .../cve-timed-task/emailtask/sending_task.go | 4 +- .../cve-timed-task/gitwebtask/gene_group.go | 19 +-- cve-vulner-manager/cve-timed-task/start.go | 14 +- .../cve-timed-task/tabletask/common.go | 22 +-- .../cve-timed-task/tabletask/crawltask.go | 1 - .../tabletask/export_excel_task.go | 6 +- .../cve-timed-task/tabletask/gauss_yaml.go | 59 ++++---- .../tabletask/import_excel_task.go | 31 ++-- .../cve-timed-task/tabletask/mapping_task.go | 31 ++-- .../tabletask/mindspore_yaml.go | 56 ++++---- .../tabletask/package_committer_task.go | 18 +-- .../cve-timed-task/tabletask/repeat_task.go | 11 +- .../cve-timed-task/tabletask/run_task.go | 45 +++--- .../tabletask/spec_error_task.go | 17 +-- .../tabletask/supplement_cve.go | 65 ++++----- .../cve-timed-task/tabletask/to_excel.go | 6 +- cve-vulner-manager/main.go | 1 + 21 files changed, 275 insertions(+), 284 deletions(-) delete mode 100644 cve-vulner-manager/cve-timed-task/db_models/init_db.go diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index fff0ba3..e62b8c5 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -46,7 +46,7 @@ RunDelFileTask = "30 9 * * 0-6" RunIssueRecordTask = "30 7 * * 0-6" RunWhitelistTask = "@every 2h" RunPackageCommitterTask = "@every 1h" -SupplementCveTask = "@every 27m" +SupplementCveTask = "@every 10m" LongSupplementCveTask = "30 1 * * 0-6" ParseOpenGaussYamlTask = "30 2 * * 0-6" ParseMindsporeYamlTask = "30 3 * * 0-6" diff --git a/cve-vulner-manager/cve-timed-task/db_models/db_operations.go b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go index 065dd56..028d9f0 100644 --- a/cve-vulner-manager/cve-timed-task/db_models/db_operations.go +++ b/cve-vulner-manager/cve-timed-task/db_models/db_operations.go @@ -7,14 +7,14 @@ import ( ) // UpdateYamlOriginMark Update origin data -func UpdateYamlOriginMark() error { +func UpdateYamlOriginMark(ORM orm.Ormer) error { sqlString := "update cve_open_guss_yaml set mark_bit = 2 where mark_bit = 1" _, err := ORM.Raw(sqlString).Exec() return err } // SelectYamlOriginData Query origin data -func SelectYamlOriginData(packageName, version string) (models.OpenGussYaml, error) { +func SelectYamlOriginData(packageName, version string, ORM orm.Ormer) (models.OpenGussYaml, error) { var openGussYaml models.OpenGussYaml sqlString := "SELECT id FROM cve_open_guss_yaml WHERE package_name = ? and version = ?" err := ORM.Raw(sqlString, packageName, version).QueryRow(&openGussYaml) @@ -22,20 +22,20 @@ func SelectYamlOriginData(packageName, version string) (models.OpenGussYaml, err } // InsertYamlOriginData insert origin data -func InsertYamlOriginData(openGussYaml *models.OpenGussYaml) error { +func InsertYamlOriginData(openGussYaml *models.OpenGussYaml, ORM orm.Ormer) error { sqlString := "insert into cve_open_guss_yaml(package_name,version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) values(?,?,?,?,?,?,?,?,?)" _, err := ORM.Raw(sqlString, openGussYaml.PackageName, openGussYaml.Version, openGussYaml.OriginUrl, openGussYaml.Status, openGussYaml.CpeName, openGussYaml.CreateTime, openGussYaml.UpdateTime, openGussYaml.MarkBit, openGussYaml.Repo).Exec() return err } // UpdateYamlOriginData Update origin data -func UpdateYamlOriginData(openGussYaml *models.OpenGussYaml) (int64, error) { +func UpdateYamlOriginData(openGussYaml *models.OpenGussYaml, ORM orm.Ormer) (int64, error) { id, err := ORM.Update(openGussYaml, "OriginUrl", "Status", "CpeName", "UpdateTime", "Repo", "MarkBit") return id, err } // SelectYamlData Query data -func SelectYamlData(packageName, version string) ([]models.GitOpenEuler, int64, error) { +func SelectYamlData(packageName, version string, ORM orm.Ormer) ([]models.GitOpenEuler, int64, error) { var gitOpenEulerList []models.GitOpenEuler sqlString := "SELECT package_id, git_id FROM cve_git_open_euler WHERE package_name = ? and version = ? order by package_id desc" rowsAffected, err := ORM.Raw(sqlString, packageName, version).QueryRows(&gitOpenEulerList) @@ -43,7 +43,7 @@ func SelectYamlData(packageName, version string) ([]models.GitOpenEuler, int64, } // SelectYamlLastData Query the last data -func SelectYamlLastData() (models.GitOpenEuler, error) { +func SelectYamlLastData(ORM orm.Ormer) (models.GitOpenEuler, error) { var gitOpenEuler models.GitOpenEuler sqlString := "SELECT package_id FROM cve_git_open_euler order by git_id desc" err := ORM.Raw(sqlString).QueryRow(&gitOpenEuler) @@ -51,7 +51,7 @@ func SelectYamlLastData() (models.GitOpenEuler, error) { } // InsertYamlData insert data -func InsertYamlData(gitOpenEuler *models.GitOpenEuler) (int64, error) { +func InsertYamlData(gitOpenEuler *models.GitOpenEuler, ORM orm.Ormer) (int64, error) { sqlString := "insert into cve_git_open_euler(package_id,package_name,version,origin_url,create_time,update_time,cpe_packname,status) values(?,?,?,?,?,?,?,?)" result, err := ORM.Raw(sqlString, gitOpenEuler.PackageId, gitOpenEuler.PackageName, gitOpenEuler.Version, gitOpenEuler.OriginUrl, gitOpenEuler.CreateTime, gitOpenEuler.UpdateTime, gitOpenEuler.CpePackName, gitOpenEuler.Status).Exec() id, _ := result.LastInsertId() @@ -59,49 +59,49 @@ func InsertYamlData(gitOpenEuler *models.GitOpenEuler) (int64, error) { } // InsertYamlDetailData insert data details -func InsertYamlDetailData(gitPackageInfo *models.GitPackageInfo) error { +func InsertYamlDetailData(gitPackageInfo *models.GitPackageInfo, ORM orm.Ormer) error { sqlString := "insert into cve_git_package_info(git_id,package_name,version,origin_url,create_time,update_time,decription,status) values(?,?,?,?,?,?,?,?)" _, err := ORM.Raw(sqlString, gitPackageInfo.GitId, gitPackageInfo.PackageName, gitPackageInfo.Version, gitPackageInfo.OriginUrl, gitPackageInfo.CreateTime, gitPackageInfo.UpdateTime, gitPackageInfo.Decription, gitPackageInfo.Status).Exec() return err } // DeleteYamlOpenEulerDetailData delete origin data -func DeleteYamlOpenEulerDetailData(gitId int64) error { +func DeleteYamlOpenEulerDetailData(gitId int64, ORM orm.Ormer) error { sqlString := "delete from cve_git_package_info where git_id = ?" _, err := ORM.Raw(sqlString, gitId).Exec() return err } // DeleteYamlOpenEulerData delete origin data -func DeleteYamlOpenEulerData(gitId int64) error { +func DeleteYamlOpenEulerData(gitId int64, ORM orm.Ormer) error { sqlString := "delete from cve_git_open_euler where git_id = ?" _, err := ORM.Raw(sqlString, gitId).Exec() return err } // UpdateYamlData update data -func UpdateYamlData(gitOpenEuler *models.GitOpenEuler) error { +func UpdateYamlData(gitOpenEuler *models.GitOpenEuler, ORM orm.Ormer) error { sqlString := "update cve_git_open_euler set origin_url = ?,update_time = ?,cpe_packname = ?,status=? where package_id = ? and package_name = ? and version = ?" _, err := ORM.Raw(sqlString, gitOpenEuler.OriginUrl, gitOpenEuler.UpdateTime, gitOpenEuler.CpePackName, gitOpenEuler.Status, gitOpenEuler.PackageId, gitOpenEuler.PackageName, gitOpenEuler.Version).Exec() return err } // UpdateYamlDetailData Update data details -func UpdateYamlDetailData(gitPackageInfo *models.GitPackageInfo) error { +func UpdateYamlDetailData(gitPackageInfo *models.GitPackageInfo, ORM orm.Ormer) error { sqlString := "update cve_git_package_info set package_name = ?,version = ?,origin_url = ?, update_time = ?,status=? where git_id = ?" _, err := ORM.Raw(sqlString, gitPackageInfo.PackageName, gitPackageInfo.Version, gitPackageInfo.OriginUrl, gitPackageInfo.UpdateTime, gitPackageInfo.Status, gitPackageInfo.GitId).Exec() return err } // DeleteYamlOriginMark delete origin data -func DeleteYamlOriginMark() error { +func DeleteYamlOriginMark(ORM orm.Ormer) error { sqlString := "delete from cve_open_guss_yaml where mark_bit = 2" _, err := ORM.Raw(sqlString).Exec() return err } // SelectOpenEulerYamlData Query data -func SelectOpenEulerYamlData() ([]models.GitOpenEuler, int64, error) { +func SelectOpenEulerYamlData(ORM orm.Ormer) ([]models.GitOpenEuler, int64, error) { var gitOpenEulerList []models.GitOpenEuler sqlString := "SELECT package_id, git_id,package_name,version FROM cve_git_open_euler WHERE package_id >= ?" rowsAffected, err := ORM.Raw(sqlString, 10000000).QueryRows(&gitOpenEulerList) @@ -109,7 +109,7 @@ func SelectOpenEulerYamlData() ([]models.GitOpenEuler, int64, error) { } // SelectGaussYamlOriginData Query origin data -func SelectGaussYamlOriginData(packageName, version string) (models.OpenGussYaml, error) { +func SelectGaussYamlOriginData(packageName, version string, ORM orm.Ormer) (models.OpenGussYaml, error) { var openGussYaml models.OpenGussYaml sqlString := "SELECT id FROM cve_open_guss_yaml WHERE package_name = ? and version = ?" err := ORM.Raw(sqlString, packageName, version).QueryRow(&openGussYaml) @@ -117,7 +117,7 @@ func SelectGaussYamlOriginData(packageName, version string) (models.OpenGussYaml } // SelectMindSporeYamlOriginData Query origin data -func SelectMindSporeYamlOriginData(packageName, version string) (models.MindSporeYaml, error) { +func SelectMindSporeYamlOriginData(packageName, version string, ORM orm.Ormer) (models.MindSporeYaml, error) { var mindSporeYaml models.MindSporeYaml sqlString := "SELECT id FROM cve_mind_spore_yaml WHERE package_name = ? and version = ?" err := ORM.Raw(sqlString, packageName, version).QueryRow(&mindSporeYaml) @@ -125,144 +125,144 @@ func SelectMindSporeYamlOriginData(packageName, version string) (models.MindSpor } // SelectOpenLookengYamlOriginData Query origin data -func SelectOpenLookengYamlOriginData(packageName, version string) (models.OpenLookengYaml, error) { +func SelectOpenLookengYamlOriginData(packageName, version string, ORM orm.Ormer) (models.OpenLookengYaml, error) { var openLookengYaml models.OpenLookengYaml sqlString := "SELECT id FROM cve_open_lookeng_yaml WHERE package_name = ? and version = ?" err := ORM.Raw(sqlString, packageName, version).QueryRow(&openLookengYaml) return openLookengYaml, err } -func GetOneGitRepoGroups(sigName string) (models.GitRepoGroups, error) { +func GetOneGitRepoGroups(sigName string, ORM orm.Ormer) (models.GitRepoGroups, error) { var gitRepsGroups models.GitRepoGroups sqlString := "select * from cve_git_repo_groups where group_name = ?" err := ORM.Raw(sqlString, sigName).QueryRow(&gitRepsGroups) return gitRepsGroups, err } -func InsertOneGitRepoGroups(gitRepoGroups *models.GitRepoGroups) error { +func InsertOneGitRepoGroups(gitRepoGroups *models.GitRepoGroups, ORM orm.Ormer) error { sqlString := "insert into cve_git_repo_groups (group_name) values (?)" _, err := ORM.Raw(sqlString, gitRepoGroups.GroupName).Exec() return err } -func DeleteGitRepoMember(groupId int64) error { +func DeleteGitRepoMember(groupId int64, ORM orm.Ormer) error { sqlString := "delete from cve_gite_repo_member where group_id = ?" _, err := ORM.Raw(sqlString, groupId).Exec() return err } -func InsertOneGiteRepoMember(giteRepoMember *models.GiteRepoMember) error { +func InsertOneGiteRepoMember(giteRepoMember *models.GiteRepoMember, ORM orm.Ormer) error { sqlString := "insert into cve_gite_repo_member (group_id,member_name,member_type,create_time) values (?, ?, ?, ?)" _, err := ORM.Raw(sqlString, giteRepoMember.GroupId, giteRepoMember.MemberName, giteRepoMember.MemberType, giteRepoMember.CreateTime).Exec() return err } -func GetCveScoreListByBeforeDate(beforeDate string) ([]models.Score, int64, error) { +func GetCveScoreListByBeforeDate(beforeDate string, ORM orm.Ormer) ([]models.Score, int64, error) { var scoreList []models.Score sqlString := "select cve_id,cve_num from cve_score where nvd_score = 0 and update_time >= ? order by update_time desc" rowsAffected, err := ORM.Raw(sqlString, beforeDate).QueryRows(&scoreList) return scoreList, rowsAffected, err } -func GetCveVulnCenterList(cveDesc, repairTime, updateTime string) ([]models.VulnCenter, int64, error) { +func GetCveVulnCenterList(cveDesc, repairTime, updateTime string, ORM orm.Ormer) ([]models.VulnCenter, int64, error) { var vulnCenterList []models.VulnCenter sqlString := "select cve_id,cve_num from cve_vuln_center where (cve_desc = ? or repair_time = ?) and update_time >= ? order by cve_id desc" rowsAffected, err := ORM.Raw(sqlString, cveDesc, repairTime, updateTime).QueryRows(&vulnCenterList) return vulnCenterList, rowsAffected, err } -func GetCveIssueTemplate(nvdScore, nvdVector, cveBrief, updateTime string) ([]models.IssueTemplate, int64, error) { +func GetCveIssueTemplate(nvdScore, nvdVector, cveBrief, updateTime string, ORM orm.Ormer) ([]models.IssueTemplate, int64, error) { var issueTemplateList []models.IssueTemplate sqlString := "select cve_id,cve_num from cve_issue_template where (nvd_score = ? or nvd_vector = ? or cve_brief = ?) and update_time >= ? order by cve_id desc" rowsAffected, err := ORM.Raw(sqlString, nvdScore, nvdVector, cveBrief, updateTime).QueryRows(&issueTemplateList) return issueTemplateList, rowsAffected, err } -func GetOneVulnCenter(cveId, cveNum string) (models.VulnCenter, error) { +func GetOneVulnCenter(cveId, cveNum string, ORM orm.Ormer) (models.VulnCenter, error) { var VulnCenter models.VulnCenter sqlString := "select cve_num, pack_name, cve_version, cve_desc,repair_time,cve_status,cve_id from cve_vuln_center where cve_id = ? and cve_num = ? order by cve_id desc" err := ORM.Raw(sqlString, cveId, cveNum).QueryRow(&VulnCenter) return VulnCenter, err } -func UpdateVulnCenterTypeOne(vulnCenter models.VulnCenter) error { +func UpdateVulnCenterTypeOne(vulnCenter models.VulnCenter, ORM orm.Ormer) error { sqlString := "update cve_vuln_center set repair_time = ?,update_time=?, cve_status=? where cve_id=?" _, err := ORM.Raw(sqlString, vulnCenter.RepairTime, vulnCenter.UpdateTime, vulnCenter.Status, vulnCenter.CveId).Exec() return err } -func UpdateVulnCenterTypeTwo(vulnCenter models.VulnCenter) error { +func UpdateVulnCenterTypeTwo(vulnCenter models.VulnCenter, ORM orm.Ormer) error { sqlString := "update cve_vuln_center set cve_desc = ?,update_time=?, cve_status=? where cve_id=?" _, err := ORM.Raw(sqlString, vulnCenter.Description, vulnCenter.UpdateTime, vulnCenter.Status, vulnCenter.CveId).Exec() return err } -func GetCveScoreByCveIdAndCveNum(cveId int64, cveNum string) (models.Score, error) { +func GetCveScoreByCveIdAndCveNum(cveId int64, cveNum string, ORM orm.Ormer) (models.Score, error) { var score models.Score sqlString := "select id,nvd_score from cve_score where cve_id = ? and cve_num = ?" err := ORM.Raw(sqlString, cveId, cveNum).QueryRow(&score) return score, err } -func UpdateCveScore(score models.Score) error { +func UpdateCveScore(score models.Score, ORM orm.Ormer) error { sqlString := "update cve_score set nvd_score=?, n_vector_value=?, n_attack_vector=?, n_access_vector=?, n_attack_complexity=?, n_access_complexity=?, n_privilege_required=?, n_user_interaction=?, n_scope=?, n_confidentiality=?, n_integrity=?, n_availability=?, n_authentication=?, update_time=?,score_type=? where id = ?" _, err := ORM.Raw(sqlString, score.NVDScore, score.NvectorVule, score.NattackVector, score.NaccessVector, score.NattackComplexity, score.NaccessComplexity, score.NprivilegeRequired, score.NuserInteraction, score.Nscope, score.Nconfidentiality, score.Nintegrity, score.Navailability, score.Nauthentication, score.UpdateTime, score.ScoreType, score.Id).Exec() return err } -func GetIssueTemplateTypeOne(issueTemplate models.IssueTemplate) (models.IssueTemplate, error) { +func GetIssueTemplateTypeOne(issueTemplate models.IssueTemplate, ORM orm.Ormer) (models.IssueTemplate, error) { var issueTemplateResp models.IssueTemplate sqlString := "select template_id,nvd_score,nvd_vector,cve_brief from cve_issue_template where (nvd_score = ? or nvd_vector = ? or cve_brief = ?) and cve_id = ? and cve_num = ?" err := ORM.Raw(sqlString, issueTemplate.NVDScore, issueTemplate.NVDVector, issueTemplate.CveBrief, issueTemplate.CveId, issueTemplate.CveNum).QueryRow(&issueTemplateResp) return issueTemplateResp, err } -func UpdateCveIssueTemplateTypeOne(issueTemplate models.IssueTemplate) error { +func UpdateCveIssueTemplateTypeOne(issueTemplate models.IssueTemplate, ORM orm.Ormer) error { sqlString := "update cve_issue_template set nvd_score = ?, update_time=? where template_id = ?" _, err := ORM.Raw(sqlString, issueTemplate.NVDScore, issueTemplate.UpdateTime, issueTemplate.TemplateId).Exec() return err } -func UpdateCveIssueTemplateTypeTwo(issueTemplate models.IssueTemplate) error { +func UpdateCveIssueTemplateTypeTwo(issueTemplate models.IssueTemplate, ORM orm.Ormer) error { sqlString := "update cve_issue_template set nvd_vector = ?, update_time=? where template_id = ? " _, err := ORM.Raw(sqlString, issueTemplate.NVDVector, issueTemplate.UpdateTime, issueTemplate.TemplateId).Exec() return err } -func UpdateCveIssueTemplateTypeThree(issueTemplate models.IssueTemplate) error { +func UpdateCveIssueTemplateTypeThree(issueTemplate models.IssueTemplate, ORM orm.Ormer) error { sqlString := "update cve_issue_template set cve_brief = ?, update_time=? where template_id = ? " _, err := ORM.Raw(sqlString, issueTemplate.CveBrief, issueTemplate.UpdateTime, issueTemplate.TemplateId).Exec() return err } -func UpdateVulnCenter(vulnCenter models.VulnCenter) error { +func UpdateVulnCenter(vulnCenter models.VulnCenter, ORM orm.Ormer) error { sqlString := "update cve_vuln_center set update_time=?, cve_status=? where cve_id=?" _, err := ORM.Raw(sqlString, vulnCenter.UpdateTime, vulnCenter.Status, vulnCenter.CveId).Exec() return err } -func GetSpceError(cveNum, cveOwner, packName string) (models.SpecError, error) { +func GetSpceError(cveNum, cveOwner, packName string, ORM orm.Ormer) (models.SpecError, error) { var spceError models.SpecError sqlString := "select * from cve_spec_error where cve_num = ? and cve_owner = ? and pack_name = ?" err := ORM.Raw(sqlString, cveNum, cveOwner, packName).QueryRow(&spceError) return spceError, err } -func UpdateCveOriginExcel(originExcel models.OriginExcel) error { +func UpdateCveOriginExcel(originExcel models.OriginExcel, ORM orm.Ormer) error { sqlString := "update cve_origin_excel set cve_desc = ?, cve_status = ? where cve_num= ? and pack_name = ? and cve_version = ?" _, err := ORM.Raw(sqlString, originExcel.CveDesc, originExcel.CveStatus, originExcel.CveNum, originExcel.PackName, originExcel.CveVersion).Exec() return err } // UpdateMindYamlOriginMark UpdateYamlOriginMarkLookeng Update origin data -func UpdateMindYamlOriginMark() error { +func UpdateMindYamlOriginMark(ORM orm.Ormer) error { sqlString := "update cve_open_lookeng_yaml set mark_bit = 2 where mark_bit = 1" _, err := ORM.Raw(sqlString).Exec() return err } // SelectMindYamlOriginData Query origin data -func SelectMindYamlOriginData(packageName, version, repokey string) (models.OpenGussYaml, error) { +func SelectMindYamlOriginData(packageName, version, repokey string, ORM orm.Ormer) (models.OpenGussYaml, error) { var openGussYaml models.OpenGussYaml sqlString := "SELECT id FROM cve_mind_spore_yaml WHERE package_name = ? and version = ? and repo_name = ?" err := ORM.Raw(sqlString, packageName, version, repokey).QueryRow(&openGussYaml) @@ -270,64 +270,64 @@ func SelectMindYamlOriginData(packageName, version, repokey string) (models.Open } // InsertMindYamlOriginData insert origin data -func InsertMindYamlOriginData(mindSporeYaml *models.MindSporeYaml) error { +func InsertMindYamlOriginData(mindSporeYaml *models.MindSporeYaml, ORM orm.Ormer) error { sqlString := "insert into cve_mind_spore_yaml(package_name,version,origin_url, status, cpe_name,create_time,update_time, mark_bit, repo_name) values(?,?,?,?,?,?,?,?,?)" _, err := ORM.Raw(sqlString, mindSporeYaml.PackageName, mindSporeYaml.Version, mindSporeYaml.OriginUrl, mindSporeYaml.Status, mindSporeYaml.CpeName, mindSporeYaml.CreateTime, mindSporeYaml.UpdateTime, mindSporeYaml.MarkBit, mindSporeYaml.Repo).Exec() return err } // UpdateMindYamlOriginData Update origin data -func UpdateMindYamlOriginData(mindSporeYaml *models.MindSporeYaml) (int64, error) { +func UpdateMindYamlOriginData(mindSporeYaml *models.MindSporeYaml, ORM orm.Ormer) (int64, error) { id, err := ORM.Update(mindSporeYaml, "OriginUrl", "Status", "CpeName", "UpdateTime", "MarkBit") return id, err } // DeleteMindYamlOriginMark delete origin data -func DeleteMindYamlOriginMark() error { +func DeleteMindYamlOriginMark(ORM orm.Ormer) error { sqlString := "delete from cve_mind_spore_yaml where mark_bit = 2" _, err := ORM.Raw(sqlString).Exec() return err } -func SelectCveFileHashByFileName(fileName string) ([]interface{}, int64, error) { +func SelectCveFileHashByFileName(fileName string, ORM orm.Ormer) ([]interface{}, int64, error) { var list orm.ParamsList sqlString := "select file_hash from cve_file_hash where file_name = ?" num, err := ORM.Raw(sqlString, fileName).ValuesFlat(&list) return list, num, err } -func InsertCveFileHash(fileName, fileHash string) error { +func InsertCveFileHash(fileName, fileHash string, ORM orm.Ormer) error { sqlString := "insert into cve_file_hash (file_name, file_hash) values (?, ?)" _, err := ORM.Raw(sqlString, fileName, fileHash).Exec() return err } -func InsertSpecIssueAssigness(specIssueAssigness *models.SpecIssueAssigness) error { +func InsertSpecIssueAssigness(specIssueAssigness *models.SpecIssueAssigness, ORM orm.Ormer) error { sqlString := "insert ignore into cve_spec_issue_assigness (package_name, issue_assignee, status, create_time) values (?, ?, ?, ?)" _, err := ORM.Raw(sqlString, specIssueAssigness.PackageName, specIssueAssigness.Assignee, specIssueAssigness.Status, specIssueAssigness.CreateTime).Exec() return err } -func SelectIssueRepoWhiteListByPackageName(packageName string) (models.IssueRepoWhitelist, error) { +func SelectIssueRepoWhiteListByPackageName(packageName string, ORM orm.Ormer) (models.IssueRepoWhitelist, error) { var issueRepoWhiteList models.IssueRepoWhitelist sqlString := "select * from cve_issue_repo_whitelist where package_name = ?" err := ORM.Raw(sqlString, packageName).QueryRow(&issueRepoWhiteList) return issueRepoWhiteList, err } -func InsertIssueRepoWhiteList(issueRepoWhiteList *models.IssueRepoWhitelist) error { +func InsertIssueRepoWhiteList(issueRepoWhiteList *models.IssueRepoWhitelist, ORM orm.Ormer) error { sqlString := "insert into cve_issue_repo_whitelist (package_name, version, status, branchs, create_time, update_time, delete_time) values (?, ?, ?, ?, ?, ?, ?)" _, err := ORM.Raw(sqlString, issueRepoWhiteList.PackageName, issueRepoWhiteList.Version, issueRepoWhiteList.Status, issueRepoWhiteList.Branchs, issueRepoWhiteList.CreateTime, issueRepoWhiteList.UpdateTime, issueRepoWhiteList.DeleteTime).Exec() return err } -func UpdateIssueRepoWhiteList(issueRepoWhiteList models.IssueRepoWhitelist) error { +func UpdateIssueRepoWhiteList(issueRepoWhiteList models.IssueRepoWhitelist, ORM orm.Ormer) error { sqlString := "update cve_issue_repo_whitelist set status = ?, branchs = ?, update_time = ? where package_name = ? and version = ?" _, err := ORM.Raw(sqlString, issueRepoWhiteList.Status, issueRepoWhiteList.Branchs, issueRepoWhiteList.UpdateTime, issueRepoWhiteList.PackageName, issueRepoWhiteList.Version).Exec() return err } -func GetResults(status string) ([]models.IssueCreateRecord, int64, error) { +func GetResults(status string, ORM orm.Ormer) ([]models.IssueCreateRecord, int64, error) { lastMonthDate := time.Now().AddDate(0, -1, 0).Format("2006-01-02") var issueCreateRecordList []models.IssueCreateRecord sqlString := "select * from cve_issue_create_record where status = ? and create_time > ?" @@ -335,39 +335,39 @@ func GetResults(status string) ([]models.IssueCreateRecord, int64, error) { return issueCreateRecordList, rowsAffected, err } -func UpdateStatue(status string) error { +func UpdateStatue(status string, ORM orm.Ormer) error { sqlString := "update cve_issue_create_record set status = ?,update_time = ? where status = ?" _, err := ORM.Raw(sqlString, 4, time.Now().Format("2006-01-02 15:04:05"), status).Exec() return err } -func GetCveEmailListByEmailType(emailType string) ([]models.EmailList, int64, error) { +func GetCveEmailListByEmailType(emailType string, ORM orm.Ormer) ([]models.EmailList, int64, error) { var emailList []models.EmailList sqlString := "select email_name from cve_email_list where email_type=?" rowsAffected, err := ORM.Raw(sqlString, emailType).QueryRows(&emailList) return emailList, rowsAffected, err } -func SelectCvePackAgeCpeByPackName(packName string) (models.PackageCpe, error) { +func SelectCvePackAgeCpeByPackName(packName string, ORM orm.Ormer) (models.PackageCpe, error) { var packageCpe models.PackageCpe sqlString := "select * from cve_package_cpe where packname = ?" err := ORM.Raw(sqlString, packName).QueryRow(&packageCpe) return packageCpe, err } -func InsertCvePackAgeCpe(packageCpe *models.PackageCpe) error { +func InsertCvePackAgeCpe(packageCpe *models.PackageCpe, ORM orm.Ormer) error { sqlString := "insert into cve_package_cpe (packname,cpe_packname,create_time) values (?,?,?)" _, err := ORM.Raw(sqlString, packageCpe.PackName, packageCpe.CpePackName, packageCpe.CreateTime).Exec() return err } -func UpdateCvePackAgeCpe(packageCpe models.PackageCpe) error { +func UpdateCvePackAgeCpe(packageCpe models.PackageCpe, ORM orm.Ormer) error { sqlString := "update cve_package_cpe set cpe_packname=?, create_time=? where packname=?" _, err := ORM.Raw(sqlString, packageCpe.CpePackName, packageCpe.CreateTime, packageCpe.PackName).Exec() return err } -func GetCveOriginExcel() ([]models.OriginExcel, int64, error) { +func GetCveOriginExcel(ORM orm.Ormer) ([]models.OriginExcel, int64, error) { lastMonthDate := time.Now().AddDate(0, -1, 0).Format("2006-01-02") var originExcel []models.OriginExcel sqlString := "select * from cve_origin_excel where (cve_status = 3 or cve_status=4) and is_export = 1 and create_time > ?" @@ -375,81 +375,81 @@ func GetCveOriginExcel() ([]models.OriginExcel, int64, error) { return originExcel, rowsAffected, err } -func UpdateStatusTypeTwo() error { +func UpdateStatusTypeTwo(ORM orm.Ormer) error { sqlString := "update cve_origin_excel set is_export=2 where cve_status=3 or cve_status=4" _, err := ORM.Raw(sqlString).Exec() return err } -func GetCveSpecError(cveNum string) (models.SpecError, error) { +func GetCveSpecError(cveNum string, ORM orm.Ormer) (models.SpecError, error) { var specError models.SpecError sqlString := "SELECT * FROM cve_spec_error where cve_num = ?" err := ORM.Raw(sqlString, cveNum).QueryRow(&specError) return specError, err } -func InsertCveSpecError(specError *models.SpecError) (int64, error) { +func InsertCveSpecError(specError *models.SpecError, ORM orm.Ormer) (int64, error) { return ORM.Insert(specError) } -func UpdateCveSpecError(specError models.SpecError) error { +func UpdateCveSpecError(specError models.SpecError, ORM orm.Ormer) error { sqlString := "update cve_spec_error set cve_desc = ?, cve_owner = ?, cve_status = ?, pack_name = ?, update_time = ? where cve_num = ?" _, err := ORM.Raw(sqlString, specError.Description, specError.Owner, specError.Status, specError.PackName, specError.UpdateTime, specError.CveNum).Exec() return err } -func GetCveOriginExcelTypeTwo(cveNum, packName, cveVersion string) (models.OriginExcel, error) { +func GetCveOriginExcelTypeTwo(cveNum, packName, cveVersion string, ORM orm.Ormer) (models.OriginExcel, error) { var originExcel models.OriginExcel sqlString := "select * from cve_origin_excel where cve_num= ? and pack_name = ? and cve_version = ?" err := ORM.Raw(sqlString, cveNum, packName, cveVersion).QueryRow(&originExcel) return originExcel, err } -func UpdateCveOriginExcelTypeThree(originExcel models.OriginExcel) error { +func UpdateCveOriginExcelTypeThree(originExcel models.OriginExcel, ORM orm.Ormer) error { sqlString := "update cve_origin_excel set nvd_score=?, cve_level=?, cve_desc=?, repair_time=?, vector_value=?, attack_vector=?, access_vector=?, attack_complexity=?, access_complexity=?, privilege_required=?, user_interaction=?, scope=?, confidentiality=?, integrity=?, availability=?, authentication=?, cve_status=?, update_time=? where cve_num=? and pack_name = ? and cve_version = ?" _, err := ORM.Raw(sqlString, originExcel.NVDScore, originExcel.CveLevel, originExcel.CveDesc, originExcel.RepairTime, originExcel.NVDVector, originExcel.AttackVector, originExcel.AccessVector, originExcel.AttackComplexity, originExcel.AccessComplexity, originExcel.PrivilegeRequired, originExcel.UserInteraction, originExcel.Scope, originExcel.Confidentiality, originExcel.Integrity, originExcel.Availability, originExcel.Authentication, originExcel.CveStatus, originExcel.UpdateTime, originExcel.CveNum, originExcel.PackName, originExcel.CveVersion).Exec() return err } -func UpdateCveOriginExcelTypeFour(originExcel models.OriginExcel) error { +func UpdateCveOriginExcelTypeFour(originExcel models.OriginExcel, ORM orm.Ormer) error { sqlString := "update cve_origin_excel set nvd_score=?, cve_level=?, cve_desc=?, repair_time=?, vector_value=?, attack_vector=?, access_vector=?, attack_complexity=?, access_complexity=?, privilege_required=?, user_interaction=?, scope=?, confidentiality=?, integrity=?, availability=?, authentication=?, cve_status=?, update_time=?, score_type=? where cve_num=? and pack_name = ? and cve_version = ?" _, err := ORM.Raw(sqlString, originExcel.NVDScore, originExcel.CveLevel, originExcel.CveDesc, originExcel.RepairTime, originExcel.NVDVector, originExcel.AttackVector, originExcel.AccessVector, originExcel.AttackComplexity, originExcel.AccessComplexity, originExcel.PrivilegeRequired, originExcel.UserInteraction, originExcel.Scope, originExcel.Confidentiality, originExcel.Integrity, originExcel.Availability, originExcel.Authentication, originExcel.CveStatus, originExcel.UpdateTime, originExcel.ScoreType, originExcel.CveNum, originExcel.PackName, originExcel.CveVersion).Exec() return err } -func UpdateCveOriginExcelTypeFive(originExcel models.OriginExcel) error { +func UpdateCveOriginExcelTypeFive(originExcel models.OriginExcel, ORM orm.Ormer) error { sqlString := "update cve_origin_excel set cve_desc = ?, cve_status = ? where cve_num = ?" _, err := ORM.Raw(sqlString, originExcel.CveDesc, originExcel.CveStatus, originExcel.CveNum).Exec() return err } -func InsertCveOriginExcel(originExcel *models.OriginExcel) error { +func InsertCveOriginExcel(originExcel *models.OriginExcel, ORM orm.Ormer) error { sqlString := "INSERT INTO cve_origin_excel (cve_num, cve_url, cve_version, pack_name, score_type, nvd_score, cve_level, cve_desc, repair_time, vector_value, attack_vector, access_vector, attack_complexity, access_complexity, privilege_required, user_interaction, scope, confidentiality, integrity, availability, authentication, cve_status, create_time, update_time) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" _, err := ORM.Raw(sqlString, originExcel.CveNum, originExcel.CveUrl, originExcel.CveVersion, originExcel.PackName, originExcel.ScoreType, originExcel.NVDScore, originExcel.CveLevel, originExcel.CveDesc, originExcel.RepairTime, originExcel.NVDVector, originExcel.AttackVector, originExcel.AccessVector, originExcel.AttackComplexity, originExcel.AccessComplexity, originExcel.PrivilegeRequired, originExcel.UserInteraction, originExcel.Scope, originExcel.Confidentiality, originExcel.Integrity, originExcel.Availability, originExcel.Authentication, originExcel.CveStatus, originExcel.CreateTime, originExcel.UpdateTime).Exec() return err } -func SelectCveOriginExcel(scoreType string) ([]models.OriginExcel, int64, error) { +func SelectCveOriginExcel(scoreType string, ORM orm.Ormer) ([]models.OriginExcel, int64, error) { var originExcel []models.OriginExcel sqlString := "SELECT cve_num,pack_name,cve_version FROM cve_origin_excel WHERE score_type = ? OR score_type IS NULL OR cve_desc IS NULL OR nvd_score IS NULL" rowsAffected, err := ORM.Raw(sqlString, scoreType).QueryRows(&originExcel) return originExcel, rowsAffected, err } -func SelectCveVulnCenter() ([]models.VulnCenter, int64, error) { +func SelectCveVulnCenter(ORM orm.Ormer) ([]models.VulnCenter, int64, error) { var vulnCenter []models.VulnCenter sqlString := "SELECT cve_id, cve_num, repair_time FROM cve_vuln_center WHERE repair_time = '' OR repair_time IS NULL" rowsAffected, err := ORM.Raw(sqlString).QueryRows(&vulnCenter) return vulnCenter, rowsAffected, err } -func UpdateCveVulnCenter(repairTime, cveId string) error { +func UpdateCveVulnCenter(repairTime, cveId string, ORM orm.Ormer) error { sqlString := "update cve_vuln_center set repair_time = ? where cve_id = ?" _, err := ORM.Raw(sqlString, repairTime, cveId).Exec() return err } -func QueryCveByNumber(cveNum string) ([]models.VulnCenter, int64, error) { +func QueryCveByNumber(cveNum string, ORM orm.Ormer) ([]models.VulnCenter, int64, error) { var vulnCenter []models.VulnCenter sqlString := "select cve_num, pack_name, cve_version, cve_desc,repair_time,cve_status,cve_id from cve_vuln_center where cve_num = ? order by cve_id desc" rowsAffected, err := ORM.Raw(sqlString, cveNum).QueryRows(&vulnCenter) diff --git a/cve-vulner-manager/cve-timed-task/db_models/init_db.go b/cve-vulner-manager/cve-timed-task/db_models/init_db.go deleted file mode 100644 index 7692a99..0000000 --- a/cve-vulner-manager/cve-timed-task/db_models/init_db.go +++ /dev/null @@ -1,13 +0,0 @@ -package db_models - -import ( - "github.com/astaxie/beego/orm" - _ "github.com/go-sql-driver/mysql" -) - -var ORM orm.Ormer - -func InitTimedTaskDB() { - myOrm := orm.NewOrm() - ORM = myOrm -} diff --git a/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go b/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go index beebeed..2b94aed 100644 --- a/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go +++ b/cve-vulner-manager/cve-timed-task/emailtask/issue_record_email.go @@ -3,12 +3,14 @@ package emailtask import ( "cvevulner/cve-timed-task/db_models" "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" "io/ioutil" ) // Sending Query all recipients func Sending(filePath string) { - result, _, err := db_models.GetCveEmailListByEmailType("1") + ormModel := orm.NewOrm() + result, _, err := db_models.GetCveEmailListByEmailType("1", ormModel) if err != nil { logs.Error("db_models.GetCveEmailListByEmailType error:", err) return diff --git a/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go b/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go index 576852b..f9e1cbe 100644 --- a/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go +++ b/cve-vulner-manager/cve-timed-task/emailtask/sending_task.go @@ -3,11 +3,13 @@ package emailtask import ( "cvevulner/cve-timed-task/db_models" "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" "io/ioutil" ) func SendingTypeTwo(filePath string) { - result, _, err := db_models.GetCveEmailListByEmailType("1") + ormModel := orm.NewOrm() + result, _, err := db_models.GetCveEmailListByEmailType("1", ormModel) if err != nil { logs.Error("db_models.GetCveEmailListByEmailType error:", err) return diff --git a/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go b/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go index e817f41..12d4113 100644 --- a/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go +++ b/cve-vulner-manager/cve-timed-task/gitwebtask/gene_group.go @@ -14,6 +14,7 @@ import ( ) func Handle() { + ormModel := orm.NewOrm() url := "https://gitee.com/openeuler/community/tree/master/sig" doc, err := util.UrlToHTML(url) if err != nil { @@ -44,33 +45,33 @@ func Handle() { } logs.Info("open transaction") - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - _, err = db_models.GetOneGitRepoGroups(sigName) + _, err = db_models.GetOneGitRepoGroups(sigName, ormModel) if err != nil { if errors.Is(orm.ErrNoRows, err) { err = db_models.InsertOneGitRepoGroups(&models.GitRepoGroups{ GroupName: sigName, - }) + }, ormModel) if err != nil { logs.Error("db_models.InsertOneGitRepoGroups error:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } } } else { logs.Error("db_models.InsertOneGitRepoGroups error:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } } } else { - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) } @@ -93,12 +94,12 @@ func Handle() { sigsList = append(sigsList, s) } for _, s := range sigsList { - gitRepoGroup, err := db_models.GetOneGitRepoGroups(s.sigName) + gitRepoGroup, err := db_models.GetOneGitRepoGroups(s.sigName, ormModel) if err != nil { logs.Warn("db_models.GetOneGitRepoGroups error:", err.Error()) continue } - err = db_models.DeleteGitRepoMember(gitRepoGroup.GroupId) + err = db_models.DeleteGitRepoMember(gitRepoGroup.GroupId, ormModel) if err != nil { logs.Warn("db_models.DeleteGitRepoMember error:", err.Error()) continue @@ -126,7 +127,7 @@ func Handle() { MemberType: "Maintainer", CreateTime: time.Now().Format("2006-01-02 15:04:05"), } - err = db_models.InsertOneGiteRepoMember(giteRepoMember) + err = db_models.InsertOneGiteRepoMember(giteRepoMember, ormModel) if err != nil { logs.Error("db_models.InsertOneGiteRepoMember error:", err.Error()) continue diff --git a/cve-vulner-manager/cve-timed-task/start.go b/cve-vulner-manager/cve-timed-task/start.go index 39c24cb..22ddeab 100644 --- a/cve-vulner-manager/cve-timed-task/start.go +++ b/cve-vulner-manager/cve-timed-task/start.go @@ -1,17 +1,7 @@ package cve_timed_task -import ( - "cvevulner/cve-timed-task/controller" - "cvevulner/cve-timed-task/db_models" -) +import "cvevulner/cve-timed-task/controller" func StartCveTimedTask() error { - - db_models.InitTimedTaskDB() - - err := controller.StartTimerTask() - if err != nil { - return err - } - return nil + return controller.StartTimerTask() } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/common.go b/cve-vulner-manager/cve-timed-task/tabletask/common.go index c8945cd..a4cd58c 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/common.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/common.go @@ -8,8 +8,8 @@ import ( ) // DeleteYamlData delete yaml data -func DeleteYamlData() { - packagesData, rowsAffected, err := db_models.SelectOpenEulerYamlData() +func DeleteYamlData(ormModel orm.Ormer) { + packagesData, rowsAffected, err := db_models.SelectOpenEulerYamlData(ormModel) if err != nil { logs.Error("db_models.SelectOpenEulerYamlData error:", err.Error()) return @@ -17,34 +17,34 @@ func DeleteYamlData() { if rowsAffected > 0 { for _, v := range packagesData { if v.PackageId >= 10000000 { - _, errg := db_models.SelectGaussYamlOriginData(v.PackageName, v.Version) - _, errm := db_models.SelectMindSporeYamlOriginData(v.PackageName, v.Version) - _, erro := db_models.SelectOpenLookengYamlOriginData(v.PackageName, v.Version) + _, errg := db_models.SelectGaussYamlOriginData(v.PackageName, v.Version, ormModel) + _, errm := db_models.SelectMindSporeYamlOriginData(v.PackageName, v.Version, ormModel) + _, erro := db_models.SelectOpenLookengYamlOriginData(v.PackageName, v.Version, ormModel) if errors.Is(orm.ErrNoRows, errg) && errors.Is(orm.ErrNoRows, errm) && errors.Is(orm.ErrNoRows, erro) { logs.Info("ID of the currently deleted data: ", v.GitId) //open transaction - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - err = db_models.DeleteYamlOpenEulerDetailData(v.GitId) + err = db_models.DeleteYamlOpenEulerDetailData(v.GitId, ormModel) if err != nil { - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } - err = db_models.DeleteYamlOpenEulerData(v.GitId) + err = db_models.DeleteYamlOpenEulerData(v.GitId, ormModel) if err != nil { - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue diff --git a/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go index da06e58..de3d7fa 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go @@ -34,7 +34,6 @@ func Crawling(url string) (XpathList, error) { xpathList := XpathList{} if url == "" || !strings.Contains(url, "http") { - logs.Error("url is not standardized:", url) return xpathList, errors.New("url is not standardized") } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go b/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go index 80cfbf9..603f6de 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/export_excel_task.go @@ -4,11 +4,13 @@ import ( "cvevulner/cve-timed-task/db_models" "fmt" "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" "github.com/xuri/excelize/v2" ) func GenerateExcels(status, path string) { - results, rowsAffected, err := db_models.GetResults(status) + ormModel := orm.NewOrm() + results, rowsAffected, err := db_models.GetResults(status, ormModel) if err != nil { logs.Error("db_models.GetResults error:", err) return @@ -42,7 +44,7 @@ func GenerateExcels(status, path string) { logs.Error("f.SaveAs error: ", err, path) } - err = db_models.UpdateStatue(status) + err = db_models.UpdateStatue(status, ormModel) if err != nil { logs.Error("db_models.UpdateStatue error: ", err) } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go index 9b095e2..869daa9 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/gauss_yaml.go @@ -48,9 +48,9 @@ func DownloadGuessYaml() (string, error) { return yamlFileName, nil } -func StoreYamlData(yamlData map[string]map[string]string) { +func StoreYamlData(yamlData map[string]map[string]string, ormModel orm.Ormer) { if len(yamlData) > 0 { - err := db_models.UpdateYamlOriginMark() + err := db_models.UpdateYamlOriginMark(ormModel) if err != nil { logs.Error("db_models.UpdateYamlOriginMark error:", err.Error()) return @@ -61,12 +61,12 @@ func StoreYamlData(yamlData map[string]map[string]string) { } //open transaction logs.Info("open transaction") - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - originData, err := db_models.SelectYamlOriginData(k, v["version"]) + originData, err := db_models.SelectYamlOriginData(k, v["version"], ormModel) now := time.Now() if err != nil { if errors.Is(orm.ErrNoRows, err) { @@ -82,10 +82,10 @@ func StoreYamlData(yamlData map[string]map[string]string) { MarkBit: 1, Repo: "security", } - err = db_models.InsertYamlOriginData(openGuessYaml) + err = db_models.InsertYamlOriginData(openGuessYaml, ormModel) if err != nil { logs.Error("db_models.InsertYamlOriginData error:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -94,7 +94,7 @@ func StoreYamlData(yamlData map[string]map[string]string) { logs.Info("insert OpenGussYaml success") } else { logs.Error("db_models.SelectYamlOriginData error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -111,20 +111,20 @@ func StoreYamlData(yamlData map[string]map[string]string) { Repo: "security", MarkBit: 1, } - _, err = db_models.UpdateYamlOriginData(openGuessYaml) + _, err = db_models.UpdateYamlOriginData(openGuessYaml, ormModel) if err != nil { logs.Error("db_models.UpdateYamlOriginData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } } - packAgesData, rowsAffected, err := db_models.SelectYamlData(k, v["version"]) + packAgesData, rowsAffected, err := db_models.SelectYamlData(k, v["version"], ormModel) if err != nil { logs.Error("db_models.SelectYamlData error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", err.Error()) } @@ -133,13 +133,13 @@ func StoreYamlData(yamlData map[string]map[string]string) { if rowsAffected == 0 { logs.Warn("No record found, Create a record") var packId int64 - gitOpenEuler, err := db_models.SelectYamlLastData() + gitOpenEuler, err := db_models.SelectYamlLastData(ormModel) if err != nil { if errors.Is(orm.ErrNoRows, err) { packId = 10000000 } else { logs.Error("db_models.SelectYamlLastData error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -162,10 +162,10 @@ func StoreYamlData(yamlData map[string]map[string]string) { CpePackName: v["cpeName"], Status: 1, } - lastId, err := db_models.InsertYamlData(goe) + lastId, err := db_models.InsertYamlData(goe, ormModel) if err != nil { logs.Error("db_models.InsertYamlData error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", err.Error()) } @@ -181,10 +181,10 @@ func StoreYamlData(yamlData map[string]map[string]string) { Decription: "", Status: 0, } - err = db_models.InsertYamlDetailData(gpi) + err = db_models.InsertYamlDetailData(gpi, ormModel) if err != nil { logs.Error("db_models.InsertYamlDetailData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", err.Error()) } @@ -193,19 +193,19 @@ func StoreYamlData(yamlData map[string]map[string]string) { } else { if rowsAffected > 1 { for _, pv := range packAgesData[1:] { - err = db_models.DeleteYamlOpenEulerDetailData(pv.GitId) + err = db_models.DeleteYamlOpenEulerDetailData(pv.GitId, ormModel) if err != nil { logs.Error("db_models.DeleteYamlOpenEulerDetailData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } - err = db_models.DeleteYamlOpenEulerData(pv.GitId) + err = db_models.DeleteYamlOpenEulerData(pv.GitId, ormModel) if err != nil { logs.Error("db_models.DeleteYamlOpenEulerData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -222,10 +222,10 @@ func StoreYamlData(yamlData map[string]map[string]string) { PackageName: k, Version: v["version"], } - err = db_models.UpdateYamlData(goe) + err = db_models.UpdateYamlData(goe, ormModel) if err != nil { logs.Error("db_models.UpdateYamlData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -239,10 +239,10 @@ func StoreYamlData(yamlData map[string]map[string]string) { Status: 0, GitId: packAgesData[0].GitId, } - err = db_models.UpdateYamlDetailData(gpi) + err = db_models.UpdateYamlDetailData(gpi, ormModel) if err != nil { logs.Error("db_models.UpdateYamlDetailData", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -250,14 +250,14 @@ func StoreYamlData(yamlData map[string]map[string]string) { } } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue } logs.Info("Transaction committed successfully", k) } - err = db_models.DeleteYamlOriginMark() + err = db_models.DeleteYamlOriginMark(ormModel) if err != nil { logs.Error("db_models.DeleteYamlOriginMark error:", err.Error()) } @@ -272,6 +272,7 @@ func StoreYamlData(yamlData map[string]map[string]string) { //3. Parse yaml //4. Save in mysql func ProcGaussYaml() { + ormModel := orm.NewOrm() yamlFileName, err := DownloadGuessYaml() if err != nil { logs.Error("DownloadGuessYaml error:", err.Error()) @@ -284,8 +285,8 @@ func ProcGaussYaml() { logs.Error("util.ParseYaml error:", yamlFileName, err) return } - StoreYamlData(yamlData) + StoreYamlData(yamlData, ormModel) - DeleteYamlData() + DeleteYamlData(ormModel) } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go b/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go index ea56900..3e20d4b 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/import_excel_task.go @@ -18,6 +18,7 @@ import ( ) func ImportDataTypeTwo() { + ormModel := orm.NewOrm() filePath, err := downloadfiletask.HandleThree() if err != nil { logs.Error("downloadfiletask.HandleThree error:", err) @@ -52,15 +53,15 @@ func ImportDataTypeTwo() { sum := hash.Sum(nil) hashValue := hex.EncodeToString(sum) logs.Info("open transaction") - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName, ormModel) if err != nil { logs.Error("db_models.SelectCveFileHashByFileName :", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -68,23 +69,23 @@ func ImportDataTypeTwo() { } if rowsAffected > 0 && util.InSlice(list, hashValue) { logs.Info("file has been parsed", fileName) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } _ = os.Remove(filePath + fileName) continue } - err = db_models.InsertCveFileHash(fileName, hashValue) + err = db_models.InsertCveFileHash(fileName, hashValue, ormModel) if err != nil { logs.Error("db_models.InsertCveFileHash :", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue @@ -133,13 +134,13 @@ func ImportDataTypeTwo() { } else { version = "" } - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - _, err = db_models.SelectIssueRepoWhiteListByPackageName(row[0]) + _, err = db_models.SelectIssueRepoWhiteListByPackageName(row[0], ormModel) if err != nil { if errors.Is(orm.ErrNoRows, err) { err = db_models.InsertIssueRepoWhiteList(&models.IssueRepoWhitelist{ @@ -148,10 +149,10 @@ func ImportDataTypeTwo() { Status: status, Branchs: branchs, CreateTime: now, - }) + }, ormModel) if err != nil { logs.Error("db_models.InsertIssueRepoWhiteList:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -159,7 +160,7 @@ func ImportDataTypeTwo() { } } else { logs.Error("db_models.SelectIssueRepoWhiteListByPackageName error", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -172,17 +173,17 @@ func ImportDataTypeTwo() { UpdateTime: now, PackageName: packName, Version: version, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateIssueRepoWhiteList error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue diff --git a/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go b/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go index a3dd6a6..0c9fade 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/mapping_task.go @@ -19,6 +19,7 @@ import ( // ToMysql Data stored in the database func ToMysql() { + ormModel := orm.NewOrm() filePath, err := downloadfiletask.HandleTwo() if err != nil { logs.Error("downloadfiletask.HandleTwo error:", err) @@ -54,15 +55,15 @@ func ToMysql() { sum := hash.Sum(nil) hashValue := hex.EncodeToString(sum) - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName, ormModel) if err != nil { logs.Error("db_models.SelectCveFileHashByFileName :", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error("Rollback transaction error:", tranErr.Error()) } @@ -71,7 +72,7 @@ func ToMysql() { if rowsAffected > 0 && util.InSlice(list, hashValue) { logs.Info("file has been parsed", fileName) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error("Rollback transaction error:", tranErr.Error()) } @@ -95,16 +96,16 @@ func ToMysql() { continue } - err = db_models.InsertCveFileHash(fileName, hashValue) + err = db_models.InsertCveFileHash(fileName, hashValue, ormModel) if err != nil { logs.Error("db_models.InsertCveFileHash :", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue @@ -138,22 +139,22 @@ func ToMysql() { cpePackName = "" } - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - packageCpe, err := db_models.SelectCvePackAgeCpeByPackName(packName) + packageCpe, err := db_models.SelectCvePackAgeCpeByPackName(packName, ormModel) if err != nil { if errors.Is(orm.ErrNoRows, err) { err := db_models.InsertCvePackAgeCpe(&models.PackageCpe{ PackName: packName, CpePackName: cpePackName, CreateTime: now, - }) + }, ormModel) if err != nil { logs.Error("db_models.InsertCvePackAgeCpe error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -161,7 +162,7 @@ func ToMysql() { } } else { logs.Error("db_models.SelectCvePackAgeCpeByPackName error: ", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -173,10 +174,10 @@ func ToMysql() { CpePackName: cpePackName, CreateTime: now, PackName: packageCpe.PackName, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCvePackAgeCpe error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -184,7 +185,7 @@ func ToMysql() { } } } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue diff --git a/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go b/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go index 4bbe08b..0e8efcd 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/mindspore_yaml.go @@ -50,9 +50,9 @@ func DownloadMindSporeYaml() (string, error) { // StoreOpenLookengYaml parse dict //Store yaml data to mysql -func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { +func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string, ormModel orm.Ormer) { if len(yamlData) > 0 { - err := db_models.UpdateMindYamlOriginMark() + err := db_models.UpdateMindYamlOriginMark(ormModel) if err != nil { logs.Error("db_models.UpdateYamlOriginMarkLookeng:", err.Error()) return @@ -64,12 +64,12 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { } //open transaction logs.Info("open transaction") - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - originData, err := db_models.SelectMindYamlOriginData(k, v["version"], RepoKey) + originData, err := db_models.SelectMindYamlOriginData(k, v["version"], RepoKey, ormModel) now := time.Now() if err != nil { if errors.Is(orm.ErrNoRows, err) { @@ -85,10 +85,10 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { MarkBit: 1, Repo: RepoKey, } - err = db_models.InsertMindYamlOriginData(mindSporeYaml) + err = db_models.InsertMindYamlOriginData(mindSporeYaml, ormModel) if err != nil { logs.Error("db_models.InsertMindYamlOriginData error:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -97,7 +97,7 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { logs.Info("insert MindSporeYaml success") } else { logs.Error("db_models.SelectMindYamlOriginData error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -113,20 +113,20 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { UpdateTime: now.Format("2006-01-02 15:04:05"), MarkBit: 1, } - _, err = db_models.UpdateMindYamlOriginData(mindSporeYaml) + _, err = db_models.UpdateMindYamlOriginData(mindSporeYaml, ormModel) if err != nil { logs.Error("db_models.UpdateMindYamlOriginData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } } - packAgesData, rowsAffected, err := db_models.SelectYamlData(k, v["version"]) + packAgesData, rowsAffected, err := db_models.SelectYamlData(k, v["version"], ormModel) if err != nil { logs.Error("db_models.SelectMindYamlData error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", err.Error()) } @@ -135,13 +135,13 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { if rowsAffected == 0 { logs.Warn("No record found, Create a record") var packId int64 - gitOpenEuler, err := db_models.SelectYamlLastData() + gitOpenEuler, err := db_models.SelectYamlLastData(ormModel) if err != nil { if errors.Is(orm.ErrNoRows, err) { packId = 10000000 } else { logs.Error("db_models.SelectYamlLastData error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -167,7 +167,7 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { lastId, err := models.InsertYamlData(goe) if err != nil { logs.Error("db_models.InsertYamlData error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", err.Error()) } @@ -183,10 +183,10 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { Decription: "", Status: 0, } - err = db_models.InsertYamlDetailData(gpi) + err = db_models.InsertYamlDetailData(gpi, ormModel) if err != nil { logs.Error("db_models.InsertYamlDetailData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", err.Error()) } @@ -195,19 +195,19 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { } else { if rowsAffected > 1 { for _, pv := range packAgesData[1:] { - err = db_models.DeleteYamlOpenEulerDetailData(pv.GitId) + err = db_models.DeleteYamlOpenEulerDetailData(pv.GitId, ormModel) if err != nil { logs.Error("db_models.DeleteYamlOpenEulerDetailData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } - err = db_models.DeleteYamlOpenEulerData(pv.GitId) + err = db_models.DeleteYamlOpenEulerData(pv.GitId, ormModel) if err != nil { logs.Error("db_models.DeleteYamlOpenEulerData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -227,7 +227,7 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { err = models.UpdateYamlData(goe) if err != nil { logs.Error("db_models.UpdateYamlData:", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -241,17 +241,17 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { Status: 0, GitId: packAgesData[0].GitId, } - err = db_models.UpdateYamlDetailData(gpi) + err = db_models.UpdateYamlDetailData(gpi, ormModel) if err != nil { logs.Error("db_models.UpdateYamlDetailData", err.Error()) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue @@ -259,7 +259,7 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { logs.Info("Transaction committed successfully", k) } } - err = db_models.DeleteMindYamlOriginMark() + err = db_models.DeleteMindYamlOriginMark(ormModel) if err != nil { logs.Error("db_models.DeleteMindYamlOriginMark error:", err.Error()) } @@ -275,6 +275,7 @@ func StoreMindSporeYaml(yamlData map[string]map[string]map[string]string) { //3. Parse yaml; //4. Save in mysql; func ProcMindSporeYaml() { + ormModel := orm.NewOrm() yamlFileName, err := DownloadMindSporeYaml() if err != nil { logs.Error("DownloadMindSporeYaml error:", err.Error()) @@ -285,8 +286,7 @@ func ProcMindSporeYaml() { logs.Error("util.ParseYamlTypeTwo error: ", yamlFileName, err) return } - StoreMindSporeYaml(yamlData) - - DeleteYamlData() + StoreMindSporeYaml(yamlData, ormModel) + DeleteYamlData(ormModel) } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go b/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go index bc24825..7a3f215 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/package_committer_task.go @@ -8,6 +8,7 @@ import ( "cvevulner/models" "encoding/hex" "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" "github.com/xuri/excelize/v2" "io" "os" @@ -16,6 +17,7 @@ import ( // import excel func ImportData() { + ormModel := orm.NewOrm() filePath, err := downloadfiletask.HandleFour() if err != nil { logs.Error("downloadfiletask.HandleFour error:", err) @@ -50,15 +52,15 @@ func ImportData() { sum := hash.Sum(nil) hashValue := hex.EncodeToString(sum) logs.Info("open transaction") - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName, ormModel) if err != nil { logs.Error("db_models.SelectCveFileHashByFileName :", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -66,23 +68,23 @@ func ImportData() { } if rowsAffected > 0 && util.InSlice(list, hashValue) { logs.Info("file has been parsed", fileName) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } _ = os.Remove(filePath + fileName) continue } - err = db_models.InsertCveFileHash(fileName, hashValue) + err = db_models.InsertCveFileHash(fileName, hashValue, ormModel) if err != nil { logs.Error("db_models.InsertCveFileHash :", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue @@ -119,7 +121,7 @@ func ImportData() { Assignee: assignee, Status: 1, CreateTime: now, - }) + }, ormModel) if err != nil { logs.Error(err) continue diff --git a/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go b/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go index 35ce2f2..d18cf0c 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/repeat_task.go @@ -5,6 +5,7 @@ import ( "cvevulner/models" "fmt" "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" "strconv" "time" ) @@ -12,9 +13,10 @@ import ( // RepeatTask Parse the artificial CVE table, crawl the CVE official // website data, and store it in the database func RepeatTask() { + ormModel := orm.NewOrm() logs.Info("CVE with empty CVE official website data is re-crawled ") - originExcel, rowsAffected, err := db_models.SelectCveOriginExcel("") + originExcel, rowsAffected, err := db_models.SelectCveOriginExcel("", ormModel) if err != nil { logs.Error("db_models.SelectCveOriginExcel error:", err) return @@ -57,7 +59,7 @@ func RepeatTask() { CveNum: v.CveNum, PackName: v.PackName, CveVersion: v.CveVersion, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveOriginExcelTypeFour error:", err) } @@ -66,8 +68,9 @@ func RepeatTask() { // GetPublishedDateTask Get the release date of nvd func GetPublishedDateTask() { + ormModel := orm.NewOrm() logs.Info("Get the release date of nvd") - vulnCenter, rowsAffected, err := db_models.SelectCveVulnCenter() + vulnCenter, rowsAffected, err := db_models.SelectCveVulnCenter(ormModel) if err != nil { logs.Error("db_models.SelectCveVulnCenter error:", err) return @@ -83,7 +86,7 @@ func GetPublishedDateTask() { continue } if crawlList.RepairTime != "" { - err = db_models.UpdateCveVulnCenter(crawlList.RepairTime, strconv.FormatInt(v.CveId, 10)) + err = db_models.UpdateCveVulnCenter(crawlList.RepairTime, strconv.FormatInt(v.CveId, 10), ormModel) if err != nil { logs.Error("db_models.UpdateCveVulnCenter error:", err) continue diff --git a/cve-vulner-manager/cve-timed-task/tabletask/run_task.go b/cve-vulner-manager/cve-timed-task/tabletask/run_task.go index 156b62c..4aace69 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/run_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/run_task.go @@ -20,6 +20,7 @@ import ( // HandleDate CVSS official website data crawling data storage database func HandleDate() { + ormModel := orm.NewOrm() filePath, err := downloadfiletask.HandleOne() if err != nil { logs.Error("downloadfiletask.HandleOne error:", err) @@ -54,15 +55,15 @@ func HandleDate() { sum := hash.Sum(nil) hashValue := hex.EncodeToString(sum) logs.Info("open transaction") - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName) + list, rowsAffected, err := db_models.SelectCveFileHashByFileName(fileName, ormModel) if err != nil { logs.Error("db_models.SelectCveFileHashByFileName :", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -71,23 +72,23 @@ func HandleDate() { fmt.Println(hashValue) if rowsAffected > 0 && util.InSlice(list, hashValue) { logs.Info("file has been parsed", fileName) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } _ = os.Remove(filePath + fileName) continue } - err = db_models.InsertCveFileHash(fileName, hashValue) + err = db_models.InsertCveFileHash(fileName, hashValue, ormModel) if err != nil { logs.Error("db_models.InsertCveFileHash :", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } continue } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue @@ -127,19 +128,19 @@ func HandleDate() { } logs.Info("begin transaction") - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - resultDict, err := db_models.GetCveOriginExcelTypeTwo(cveNum, packName, cveVersion) + resultDict, err := db_models.GetCveOriginExcelTypeTwo(cveNum, packName, cveVersion, ormModel) if err != nil { if errors.Is(orm.ErrNoRows, err) { url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) crawlList, err := Crawling(url) if err != nil { logs.Error("Crawling error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error("Rollback transaction error:", tranErr.Error()) } @@ -174,10 +175,10 @@ func HandleDate() { CveStatus: 0, CreateTime: now, UpdateTime: now, - }) + }, ormModel) if err != nil { logs.Error("db_models.InsertCveOriginExcel error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error("Rollback transaction error:", tranErr.Error()) } @@ -186,7 +187,7 @@ func HandleDate() { } else { logs.Error("db_models.GetCveOriginExcelTypeTwo error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error("Rollback transaction error:", tranErr.Error()) } @@ -197,7 +198,7 @@ func HandleDate() { crawlList, err := Crawling(url) if err != nil { logs.Error("Crawling error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error("Rollback transaction error:", tranErr.Error()) } @@ -235,10 +236,10 @@ func HandleDate() { CveNum: cveNum, PackName: packName, CveVersion: cveVersion, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveOriginExcelTypeThree error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error("Rollback transaction error:", tranErr.Error()) } @@ -272,10 +273,10 @@ func HandleDate() { CveNum: cveNum, PackName: packName, CveVersion: cveVersion, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveOriginExcelTypeFour error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error("Rollback transaction error:", tranErr.Error()) } @@ -287,7 +288,7 @@ func HandleDate() { } } - specError, err := db_models.GetCveSpecError(cveNum) + specError, err := db_models.GetCveSpecError(cveNum, ormModel) if err != nil { if !errors.Is(orm.ErrNoRows, err) { logs.Error("db_models.GetCveSpecError error:", err) @@ -297,17 +298,17 @@ func HandleDate() { CveDesc: specError.Description, CveStatus: 6, CveNum: cveNum, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveOriginExcelTypeFive error:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error("Rollback transaction error:", tranErr.Error()) } continue } } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue diff --git a/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go b/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go index 6f25f90..5dc0765 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go @@ -14,6 +14,7 @@ import ( // AddErrorDetails Conditions for filtering incorrect data func AddErrorDetails() { + ormModel := orm.NewOrm() filePath, err := downloadfiletask.DownloadSpecErrorExcels() if err != nil { logs.Error("downloadfiletask.DownloadSpecErrorExcels error:", err) @@ -71,12 +72,12 @@ func AddErrorDetails() { } else { cveOwner = "" } - tranErr := db_models.ORM.Begin() + tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) continue } - specError, err := db_models.GetCveSpecError(cveNum) + specError, err := db_models.GetCveSpecError(cveNum, ormModel) if err != nil { if errors.Is(orm.ErrNoRows, err) { _, err = db_models.InsertCveSpecError(&models.SpecError{ @@ -86,10 +87,10 @@ func AddErrorDetails() { Status: 1, PackName: packName, CreateTime: now, - }) + }, ormModel) if err != nil { logs.Error("db_models.InsertCveSpecError:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -97,7 +98,7 @@ func AddErrorDetails() { } } else { logs.Error("db_models.GetCveSpecError:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -114,10 +115,10 @@ func AddErrorDetails() { PackName: packName, Status: 1, UpdateTime: now, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveSpecError:", err) - tranErr = db_models.ORM.Rollback() + tranErr = ormModel.Rollback() if tranErr != nil { logs.Error(" Rollback transaction error:", tranErr.Error()) } @@ -125,7 +126,7 @@ func AddErrorDetails() { } } } - tranErr = db_models.ORM.Commit() + tranErr = ormModel.Commit() if tranErr != nil { logs.Error(" Commit transaction error:", tranErr.Error()) continue diff --git a/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go index e436b0c..b50349c 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go @@ -21,12 +21,12 @@ type Compare struct { } // QueryCveALLData Find cve with missing necessary fields -func QueryCveALLData(days int) []models.VulnCenter { +func QueryCveALLData(days int, ormModel orm.Ormer) []models.VulnCenter { now := time.Now() beforeDate := time.Date(now.Year(), now.Month(), now.Day()-days, 0, 0, 0, 0, now.Location()).Format("2006-01-02 15:04:05") cveIdList := make([]Compare, 0, 100) - scoreList, rowsAffected, err := db_models.GetCveScoreListByBeforeDate(beforeDate) + scoreList, rowsAffected, err := db_models.GetCveScoreListByBeforeDate(beforeDate, ormModel) if err == nil { if rowsAffected > 0 { for _, v := range scoreList { @@ -35,13 +35,11 @@ func QueryCveALLData(days int) []models.VulnCenter { CveNum: v.CveNum, }) } - } else { - logs.Info("No query results") } } else { logs.Error("db_models.GetCveScoreList error:", err.Error()) } - vulnCenterList, rowsAffected, err := db_models.GetCveVulnCenterList("", "", beforeDate) + vulnCenterList, rowsAffected, err := db_models.GetCveVulnCenterList("", "", beforeDate, ormModel) if err == nil { if rowsAffected > 0 { for _, v := range vulnCenterList { @@ -50,14 +48,12 @@ func QueryCveALLData(days int) []models.VulnCenter { CveNum: v.CveNum, }) } - } else { - logs.Info("No query results") } } else { logs.Error("db_models.GetCveVulnCenter error:", err.Error()) } - issueTemplateList, rowsAffected, err := db_models.GetCveIssueTemplate("0", "", "", beforeDate) + issueTemplateList, rowsAffected, err := db_models.GetCveIssueTemplate("0", "", "", beforeDate, ormModel) if err == nil { if rowsAffected > 0 { for _, v := range issueTemplateList { @@ -66,10 +62,7 @@ func QueryCveALLData(days int) []models.VulnCenter { CveNum: v.CveNum, }) } - } else { - logs.Info("No query results") } - } else { logs.Error("db_models.GetCveIssueTemplate error:", err.Error()) } @@ -78,7 +71,7 @@ func QueryCveALLData(days int) []models.VulnCenter { cveList := make([]models.VulnCenter, 0, len(cveIdList)) if len(cveIdList) > 0 { for _, v := range cveIdList { - vulnCenter, err := db_models.GetOneVulnCenter(v.CveId, v.CveNum) + vulnCenter, err := db_models.GetOneVulnCenter(v.CveId, v.CveNum, ormModel) if err != nil { logs.Error("db_models.GetOneVulnCenter error:", err.Error()) continue @@ -108,7 +101,7 @@ func RemoveRepeatedElement(arr []Compare) (newArr []Compare) { } // UpdateCveVuln update data -func UpdateCveVuln(url string, cve models.VulnCenter) error { +func UpdateCveVuln(url string, cve models.VulnCenter, ormModel orm.Ormer) error { cveNum := strings.TrimSpace(cve.CveNum) cveId := cve.CveId cveStatus := cve.Status @@ -149,7 +142,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { UpdateTime: updateTime, Status: cveStatusx, CveId: cveId, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateVulnCenterTypeOne error:", err) return err @@ -162,7 +155,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { UpdateTime: updateTime, Status: cveStatusx, CveId: cveId, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateVulnCenterTypeTwo error:", err) return err @@ -180,7 +173,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { } else { scoreType = "v2" } - score, err := db_models.GetCveScoreByCveIdAndCveNum(cveId, cveNum) + score, err := db_models.GetCveScoreByCveIdAndCveNum(cveId, cveNum, ormModel) if err != nil { if errors.Is(err, orm.ErrNoRows) { logs.Info(err) @@ -207,7 +200,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { UpdateTime: updateTime, ScoreType: scoreType, Id: score.Id, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveScore error:", err) return err @@ -222,7 +215,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { CveBrief: "", CveId: cveId, CveNum: cveNum, - }) + }, ormModel) if err != nil { logs.Error(" db_models.GetIssueTemplateTypeOne error:", err) return nil @@ -234,7 +227,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { NVDScore: nvdScore, UpdateTime: updateTime, TemplateId: templateResult.TemplateId, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveIssueTemplateTypeOne error:", err) return err @@ -247,7 +240,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { NVDVector: listx.VectorValue, UpdateTime: updateTime, TemplateId: templateResult.TemplateId, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveIssueTemplateTypeTwo error:", err) return err @@ -260,7 +253,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { CveBrief: listx.CveDesc, UpdateTime: updateTime, TemplateId: templateResult.TemplateId, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveIssueTemplateTypeThree error:", err) return err @@ -273,7 +266,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { UpdateTime: updateTime, Status: cveStatusx, CveId: cveId, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateVulnCenter error:", err) return err @@ -287,8 +280,8 @@ func UpdateCveVuln(url string, cve models.VulnCenter) error { //2. Go to the CVE official website to find the CVE information; //3. Fill in the corresponding table again; func SupplementCve() { - - cveList := QueryCveALLData(30) + ormModel := orm.NewOrm() + cveList := QueryCveALLData(3, ormModel) if len(cveList) > 0 { for _, cve := range cveList { @@ -296,8 +289,8 @@ func SupplementCve() { cveVersion := strings.TrimSpace(cve.CveVersion) packName := strings.TrimSpace(cve.PackName) url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) - _ = UpdateCveVuln(url, cve) - resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName) + _ = UpdateCveVuln(url, cve, ormModel) + resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName, ormModel) if err != nil { logs.Error("db_models.GetSpceError error:", err) continue @@ -309,7 +302,7 @@ func SupplementCve() { CveNum: cveNum, PackName: packName, CveVersion: cveVersion, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveOriginExcel error:", err) } @@ -321,15 +314,16 @@ func SupplementCve() { //2. Go to the CVE official website to find the CVE information; //3. Fill in the corresponding table again; func LongSupplementCve() { - cveList := QueryCveALLData(1500) + ormModel := orm.NewOrm() + cveList := QueryCveALLData(1500, ormModel) if len(cveList) > 0 { for _, cve := range cveList { cveNum := strings.TrimSpace(cve.CveNum) cveVersion := strings.TrimSpace(cve.CveVersion) packName := strings.TrimSpace(cve.PackName) url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) - _ = UpdateCveVuln(url, cve) - resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName) + _ = UpdateCveVuln(url, cve, ormModel) + resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName, ormModel) if err != nil { logs.Error("db_models.GetSpceError error:", err) continue @@ -341,7 +335,7 @@ func LongSupplementCve() { CveNum: cveNum, PackName: packName, CveVersion: cveVersion, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveOriginExcel error:", err) } @@ -350,8 +344,9 @@ func LongSupplementCve() { } func PullCve(cveNum string) (resp int) { + ormModel := orm.NewOrm() resp = 0 - cveList, rowsAffected, err := db_models.QueryCveByNumber(cveNum) + cveList, rowsAffected, err := db_models.QueryCveByNumber(cveNum, ormModel) if err != nil { return resp } @@ -366,11 +361,11 @@ func PullCve(cveNum string) (resp int) { cveVersion := strings.TrimSpace(cve.CveVersion) packName := strings.TrimSpace(cve.PackName) url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) - err = UpdateCveVuln(url, cve) + err = UpdateCveVuln(url, cve, ormModel) if err == nil { resp = 1 } - resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName) + resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName, ormModel) if err != nil { logs.Error("db_models.GetSpceError error:", err) } @@ -381,7 +376,7 @@ func PullCve(cveNum string) (resp int) { CveNum: cveNum, PackName: packName, CveVersion: cveVersion, - }) + }, ormModel) if err != nil { logs.Error("db_models.UpdateCveOriginExcel error:", err) } diff --git a/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go b/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go index b3cc56e..7a0353b 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/to_excel.go @@ -4,11 +4,13 @@ import ( "cvevulner/cve-timed-task/db_models" "fmt" "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" "github.com/xuri/excelize/v2" ) func GenerateTable(path string) { - results, rowsAffected, err := db_models.GetCveOriginExcel() + ormModel := orm.NewOrm() + results, rowsAffected, err := db_models.GetCveOriginExcel(ormModel) if err != nil { logs.Error("db_models.GetCveOriginExcel error:", err) return @@ -44,7 +46,7 @@ func GenerateTable(path string) { logs.Error("f.SaveAs error: ", err, path) } - err = db_models.UpdateStatusTypeTwo() + err = db_models.UpdateStatusTypeTwo(ormModel) if err != nil { logs.Error("db_models.UpdateStatusTypeTwo error: ", err) } diff --git a/cve-vulner-manager/main.go b/cve-vulner-manager/main.go index eb6d1a5..02bc418 100644 --- a/cve-vulner-manager/main.go +++ b/cve-vulner-manager/main.go @@ -36,6 +36,7 @@ func main() { if err != nil { return } + // single run task.StartTask() defer task.StopTask() -- Gitee From cb1143cf584ef44d428e6f9cef3e4c8327cc7c86 Mon Sep 17 00:00:00 2001 From: "775517776@qq.com" <775517776@qq.com> Date: Thu, 10 Mar 2022 15:25:17 +0800 Subject: [PATCH 7/8] =?UTF-8?q?=E2=80=98=E4=BF=AE=E6=94=B9bug=E2=80=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cve-timed-task/tabletask/crawltask.go | 6 +++--- .../cve-timed-task/tabletask/run_task.go | 1 - .../cve-timed-task/tabletask/spec_error_task.go | 2 -- .../cve-timed-task/tabletask/supplement_cve.go | 17 +++++++---------- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go index de3d7fa..92f9c8a 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/crawltask.go @@ -1,12 +1,13 @@ package tabletask import ( + "strings" + "time" + "cvevulner/cve-timed-task/util" "github.com/antchfx/htmlquery" "github.com/astaxie/beego/logs" "github.com/pkg/errors" - "strings" - "time" ) type XpathList struct { @@ -39,7 +40,6 @@ func Crawling(url string) (XpathList, error) { html, err := util.UrlToHTML(url) if err != nil { - logs.Error("util.UrlToHTML error:", err.Error(), url) return xpathList, err } //xpathList.CveDesc diff --git a/cve-vulner-manager/cve-timed-task/tabletask/run_task.go b/cve-vulner-manager/cve-timed-task/tabletask/run_task.go index 4aace69..a1acd90 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/run_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/run_task.go @@ -127,7 +127,6 @@ func HandleDate() { cveVersion = "" } - logs.Info("begin transaction") tranErr := ormModel.Begin() if tranErr != nil { logs.Error(" Open transaction error:", tranErr.Error()) diff --git a/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go b/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go index 5dc0765..9bfea1b 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/spec_error_task.go @@ -105,9 +105,7 @@ func AddErrorDetails() { continue } } else { - logs.Info("CVE recorded", cveNum) if specError.Description != cveDesc || specError.Owner != cveOwner || specError.PackName != packName || specError.Status != 1 { - logs.Info("update cve_spec_error") err := db_models.UpdateCveSpecError(models.SpecError{ CveNum: specError.CveNum, Description: cveDesc, diff --git a/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go index b50349c..2c93575 100644 --- a/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go +++ b/cve-vulner-manager/cve-timed-task/tabletask/supplement_cve.go @@ -1,16 +1,17 @@ package tabletask import ( + "fmt" + "strconv" + "strings" + "time" + "cvevulner/cve-timed-task/db_models" "cvevulner/cve-timed-task/util" "cvevulner/models" - "fmt" "github.com/astaxie/beego/logs" "github.com/astaxie/beego/orm" "github.com/pkg/errors" - "strconv" - "strings" - "time" ) //var cveListDate []db_models.VulnCenter @@ -162,10 +163,7 @@ func UpdateCveVuln(url string, cve models.VulnCenter, ormModel orm.Ormer) error } } - nvdScore, err := strconv.ParseFloat(listx.NvdScore, 64) - if err != nil { - logs.Error(err.Error()) - } + nvdScore, _ := strconv.ParseFloat(listx.NvdScore, 64) if nvdScore > 0 && listx.NvdScore != "" { var scoreType string if listx.ScoreType == "v3.0" { @@ -291,11 +289,10 @@ func SupplementCve() { url := fmt.Sprintf("https://nvd.nist.gov/vuln/detail/%s", cveNum) _ = UpdateCveVuln(url, cve, ormModel) resultSpecError, err := db_models.GetSpceError(cveNum, "src-openEuler", packName, ormModel) - if err != nil { + if err != nil && !errors.Is(orm.ErrNoRows, err) { logs.Error("db_models.GetSpceError error:", err) continue } - logs.Info("Filter, modify the status to 6", cveNum) err = db_models.UpdateCveOriginExcel(models.OriginExcel{ CveDesc: resultSpecError.Description, CveStatus: 6, -- Gitee From 5bca9b455c50ef990ea111da3cd5e7dbb16cef6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E9=98=B3?= <775517776@qq.com> Date: Tue, 29 Mar 2022 01:28:39 +0000 Subject: [PATCH 8/8] update cve-vulner-manager/conf/app.conf. --- cve-vulner-manager/conf/app.conf | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/cve-vulner-manager/conf/app.conf b/cve-vulner-manager/conf/app.conf index f70de9a..8cfa982 100644 --- a/cve-vulner-manager/conf/app.conf +++ b/cve-vulner-manager/conf/app.conf @@ -54,23 +54,12 @@ ParseMindsporeYamlTask = "30 3 * * 0-6" -; [mysql] -; dbhost = "${DB_URI||127.0.0.1}" -; dbport = 3310 -; dbuser = "${DB_USER||cve}" -; dbpwd = "${DB_PWD||***}" -; dbname = oneline_cvevulner -; dbprefix = cve_ -; maxidle = 30 -; maxconn = 3000 - - [mysql] -dbhost = "127.0.0.1" -dbport = 3306 -dbuser = "root" -dbpwd = "root" -dbname = cvevulner +dbhost = "${DB_URI||127.0.0.1}" +dbport = 3310 +dbuser = "${DB_USER||cve}" +dbpwd = "${DB_PWD||***}" +dbname = oneline_cvevulner dbprefix = cve_ maxidle = 30 maxconn = 3000 -- Gitee