From 97bc27e59e7f6fd7fc00707165772176fc3de813 Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Mon, 21 Dec 2020 19:30:54 +0800 Subject: [PATCH] 1. Manually submit cve to capture and describe the failure problem; 2. Fail to extract cve data; --- conf/product_app.conf | 6 +++- cve-py/controller/taskcontroller.py | 3 +- cve-py/main.py | 1 - cve-py/tabletask/crawltask.py | 4 +-- cve-py/tabletask/repeattask.py | 53 +++++++++++++++++++++++++++++ cve-py/tabletask/runtask.py | 7 ++-- models/issue.go | 3 +- models/oricvecheck.go | 16 +++++++++ task/inittask.go | 16 +++++++++ task/oricvecheck.go | 43 +++++++++++++++++++++-- taskhandler/cve.go | 6 +++- taskhandler/grabissue.go | 15 ++++---- taskhandler/oricvecheck.go | 40 ++++++++++++++++++++-- 13 files changed, 192 insertions(+), 21 deletions(-) create mode 100644 cve-py/tabletask/repeattask.py diff --git a/conf/product_app.conf b/conf/product_app.conf index 392410e..2bd1d35 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -61,7 +61,7 @@ gittokenflag = 2 issueoath = * * */20 * * * genexcelflag = 1 genexcel = 0 30 7 * * * -days = -30 +days = -50 prcnum = 50 printlogflag = 1 printlog = 0 0 1 * * 1 @@ -69,6 +69,8 @@ unlockflag = 1 unlock = 0 0 12 * * * hookflag = 1 prochook = 0 1 8 * * * +exceptflag = 1 +exceptcve = 0 0 4 * * * [gitee] @@ -114,6 +116,8 @@ issue_whitelist = 1 affected_branchs = openEuler-20.03-LTS # Close the highest privilege of issue close_issue_privilege = 1 +# abnormal cve status ,Use "," to separate multiple states +abn_cve_status =3 [reflink] diff --git a/cve-py/controller/taskcontroller.py b/cve-py/controller/taskcontroller.py index 561d876..e4e00f0 100644 --- a/cve-py/controller/taskcontroller.py +++ b/cve-py/controller/taskcontroller.py @@ -14,7 +14,7 @@ Date: 10/22/2020 11:01 AM """ from tabletask import (runtask, mappingtask, toexcel, export_excel_task, import_excel_task, package_committer_task, - spec_error_task) + spec_error_task, repeattask) from gitwebtask import genegroup, yamltask from emailtask import sendingtask, issue_record_email from deletetask import deletefiletask @@ -48,6 +48,7 @@ def runtabletask(): print("Analyze the manual data table, crawl the CVE official website data, and start the timing task") spec_error_task.add_error_details() runtask.handle_data() + repeattask.repeat_task() print("Analyze the manual data table, crawl the CVE official website data task completed") diff --git a/cve-py/main.py b/cve-py/main.py index 9c2213b..f988f87 100644 --- a/cve-py/main.py +++ b/cve-py/main.py @@ -17,7 +17,6 @@ Date: 10/22/2020 11:01 AM """ from controller import timertaskcontroller - if __name__ == '__main__': print("The program starts, waiting for the timing task to execute") timertaskcontroller.timertask() diff --git a/cve-py/tabletask/crawltask.py b/cve-py/tabletask/crawltask.py index f7b8aff..3a03d08 100644 --- a/cve-py/tabletask/crawltask.py +++ b/cve-py/tabletask/crawltask.py @@ -58,7 +58,7 @@ def crawling(url): access_vector = attack_complexity = access_complexity = \ privilege_required = user_interaction = scope = confidentiality = \ integrity = availability = authentication = None - print("No data on this vulnerability link") + print("No data on this vulnerability link, ", url) score_type = "" else: score_type = "v2.0" @@ -90,7 +90,7 @@ def crawling(url): access_vector = attack_complexity = access_complexity = \ privilege_required = user_interaction = scope = confidentiality = integrity = \ availability = authentication = score_type = None - print("This vulnerability link not found") + print("This vulnerability link not found, ", url) else: score_type = "v3.0" cve_desc = str(html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')[0]) diff --git a/cve-py/tabletask/repeattask.py b/cve-py/tabletask/repeattask.py new file mode 100644 index 0000000..044ffbe --- /dev/null +++ b/cve-py/tabletask/repeattask.py @@ -0,0 +1,53 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Handle failed tasks + +Authors: xiaojianghui +Date: 21/12/2020 11:01 AM +""" + +import time +from dbConnecttion.MysqlConn import Mysql +from tabletask import crawltask + + +def cur_date(): + """ + current date + Returns:createTime:string + + """ + create_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + return create_time + + +def repeat_task(): + print("CVE官网数据为空的CVE进行再次抓取") + mysql = Mysql() + sql = "SELECT cve_num FROM cve_origin_excel WHERE score_type = %s " \ + "OR score_type IS NULL OR cve_desc IS NULL OR nvd_score IS NULL" + val = ('',) + result = mysql.getMany(sql, val) + print("CVE官网数据为空的 数据：", result) + for i in result: + crawllist = crawltask.crawling("https://nvd.nist.gov/vuln/detail/" + i["cve_num"]) + print("更新CVE数据") + sql = "update cve_origin_excel set nvd_score=%s, cve_level=%s, cve_desc=%s, " \ + "repair_time=%s, vector_value=%s, attack_vector=%s, access_vector=%s, " \ + "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ + "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ + "availability=%s, authentication=%s, cve_status=%s, update_time=%s, " \ + "score_type=%s where cve_num=%s" + val = (crawllist[0], crawllist[1], crawllist[2], crawllist[3], crawllist[4], crawllist[5], crawllist[6], + crawllist[7], crawllist[8], crawllist[9], crawllist[10], crawllist[11], crawllist[12], crawllist[13], + crawllist[14], crawllist[15], 1, cur_date(), crawllist[16], i["cve_num"]) + mysql.update(sql, val) + mysql.dispose() + mysql.close() diff --git a/cve-py/tabletask/runtask.py b/cve-py/tabletask/runtask.py index ced6ddc..61451ab 100644 --- a/cve-py/tabletask/runtask.py +++ b/cve-py/tabletask/runtask.py @@ -68,6 +68,7 @@ def handle_data(): if result_dict: # Crawler web data crawl_list = crawltask.crawling(urls[i]) + print(crawl_list) # Determine whether the database content is the latest data if crawl_list[0]: if str(result_dict["nvd_score"]) == str(crawl_list[0]) and str(result_dict["vector_value"] @@ -103,13 +104,13 @@ def handle_data(): "repair_time=%s, vector_value=%s, attack_vector=%s, access_vector=%s, " \ "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ - "availability=%s, authentication=%s, cve_status=%s, update_time=%s " \ - "where cve_num=%s" + "availability=%s, authentication=%s, cve_status=%s, update_time=%s, " \ + "score_type=%s where cve_num=%s" val = ( crawl_list[0], crawl_list[1], crawl_list[2], crawl_list[3], crawl_list[4], crawl_list[5], crawl_list[6], crawl_list[7], crawl_list[8], crawl_list[9], crawl_list[10], crawl_list[11], crawl_list[12], crawl_list[13], crawl_list[14], - crawl_list[15], 1, update_time, cve_num) + crawl_list[15], 1, update_time, crawl_list[16], cve_num) mysql.update(sql, val) mysql.dispose() except IndexError as e: diff --git a/models/issue.go b/models/issue.go index 1897b58..72c4677 100644 --- a/models/issue.go +++ b/models/issue.go @@ -168,12 +168,13 @@ func GetSpecIssueAssignee(sia *SpecIssueAssigness, colName ...string) error { } func UpdateIssueTemplate(it *IssueTemplate, fields ...string) error { - logs.Info("===>", it, fields) + logs.Info("Update===>", it, fields) o := orm.NewOrm() _, err := o.Update(it, fields...) return err } + func UpdateScore(s *Score, fields ...string) error { o := orm.NewOrm() _, err := o.Update(s, fields...) diff --git a/models/oricvecheck.go b/models/oricvecheck.go index be8f0b0..a23594d 100644 --- a/models/oricvecheck.go +++ b/models/oricvecheck.go @@ -2,6 +2,7 @@ package models import ( "cvevulner/common" + "fmt" "github.com/astaxie/beego/logs" "github.com/astaxie/beego/orm" ) @@ -54,3 +55,18 @@ func UpdateOriginExistTemp() (bool) { return false } } + +// Query abnormal cve data +func QueryAbnCve(cveSt, days string, prcnum int, cveId int64) ([]VulnCenter, error) { + o := orm.NewOrm() + var vc []VulnCenter + cveSql := fmt.Sprintf("select * from cve_vuln_center where "+ + "update_time >= '%s' and cve_id > %d and cve_status in (%s) "+ + "order by cve_id asc limit %d", days, cveId, cveSt, prcnum) + logs.Info("cveSql: ", cveSql) + num, err := o.Raw(cveSql).QueryRows(&vc) + if err == nil && num > 0 { + return vc, err + } + return vc, err +} diff --git a/task/inittask.go b/task/inittask.go index b292e22..9077f9a 100644 --- a/task/inittask.go +++ b/task/inittask.go @@ -78,6 +78,14 @@ func GenSAExcelTask(genExcel string) { logs.Info("执行issue生成excel task end") } +// Handling abnormal cve data +func AbnormalCveTask(exceptcve string) { + logs.Info("创建处理异常cve任务 task start") + procabnTask := toolbox.NewTask("AbnormalCve", exceptcve, ProcAbnCve) + toolbox.AddTask("AbnormalCve", procabnTask) + logs.Info("创建处理异常cve任务 task end") +} + // Print logs to the console and delete redundant logs func PrintLogTask(printLog string) { logs.Info("创建日志任务 task start") @@ -174,5 +182,13 @@ func InitTask() bool { } else { logs.Info("prochook no task") } + // Handling abnormal cve data + exceptflag, errx := BConfig.Int("crontab::exceptflag") + if exceptflag == 1 && errx == nil { + exceptcve := BConfig.String("crontab::exceptcve") + AbnormalCveTask(exceptcve) + } else { + logs.Info("exceptcve no task") + } return true } diff --git a/task/oricvecheck.go b/task/oricvecheck.go index 7e57b60..3765a09 100644 --- a/task/oricvecheck.go +++ b/task/oricvecheck.go @@ -7,6 +7,7 @@ import ( "errors" "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" + "strings" ) //CheckOriCve Check the CVE of the original data @@ -20,9 +21,9 @@ func CheckOriCve() error { return err } // The amount of data processed at a time - prcNum, err := BConfig.Int("crontab::prcNum") + prcNum, err := BConfig.Int("crontab::prcnum") if err != nil { - logs.Error("config crontab::prcNum error: invalid value is ", prcNum) + logs.Error("config crontab::prcnum error: invalid value is ", prcNum) return errors.New("value is nil") } // Get the data source of the table @@ -46,3 +47,41 @@ func UnLockTable() error { logs.Info("执行解锁表 task end") return nil } + + +// Cve handles abnormal data reprocessing +func ProcAbnCve() error { + defer common.Catchs() + logs.Info("纠正cve数据 task start") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return err + } + // The amount of data processed at a time + prcNum, err := BConfig.Int("crontab::prcnum") + if err != nil { + logs.Error("config crontab::prcnum error: invalid value is ", prcNum) + return errors.New("value is nil") + } + days, err := BConfig.Int("crontab::days") + if err != nil { + logs.Error("config crontab::days error: invalid value is ", days) + return errors.New("value is nil") + } + cveStatuStr := BConfig.String("cve::abn_cve_status") + if cveStatuStr == "" { + logs.Error("config cve::abn_cve_status error: invalid value is ", cveStatuStr) + return errors.New("value is nil") + } + st := strings.Split(cveStatuStr, ",") + for k, v := range st { + st[k] = "'" + v + "'" + } + cveSt := strings.Join(st, ",") + // Get the data source of the table + _, err = taskhandler.UpdateAbnCveStatus(prcNum, days, cveSt) + logs.Info("纠正cve数据 task end") + return err +} diff --git a/taskhandler/cve.go b/taskhandler/cve.go index db0b1b3..025976e 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -891,6 +891,7 @@ func GetCveOriginData(prcnum, days, openeulernum int, cveRef string) (bool, erro lockOk := models.LockUpdateOriginStatus(common.GetCurTime(), cveOrg.PackName, cveOrg.Version, cveOrg.CveId, 15) if !lockOk { logs.Error("Current data is being processed: PackName: ", cveOrg.PackName, cveOrg.Version, cveOrg.CveId) + ch <- i continue } go func(idx int, cveData models.OriginUpstream) { @@ -935,6 +936,7 @@ func GetCveOriginExcelData(prcnum, days, openeulerNum int, cveRef string) (bool, lockOk := models.LockUpdateOriginExcelStatus(common.GetCurTime(), cveOrg.CveId, 15) if !lockOk { logs.Error("Current data is being processed, cveNum: ", cveOrg.CveNum, ",cveId: ", cveOrg.CveId) + ch <- i continue } go func(idx, manYears int, cveData models.OriginExcel) { @@ -942,7 +944,7 @@ func GetCveOriginExcelData(prcnum, days, openeulerNum int, cveRef string) (bool, if !ok { logs.Error("SyncCveVuler, 处理失败, err: ", err, "cveData: ", cveData) } - ch <- i + ch <- idx }(i, manYears, cveOrg) } for i := 0; i < cap(ch); i++ { @@ -1748,6 +1750,7 @@ func GetCveIssueData(prcnum, days, openeulernum int, cveRef, owner string) (bool lockOk := models.LockUpdateCveIssueStatus(15, cveOrg.Id) if !lockOk { logs.Error("Data is being processed, id: ", cveOrg.Id, ",cveOrg: ", cveOrg) + ch <- i continue } if cveOrg.CveNumber == "" || len(cveOrg.CveNumber) == 0 { @@ -1923,6 +1926,7 @@ func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { sendError := SendEmail(zipFileName) if sendError != nil { logs.Error("SendEmail, sendErr: ", sendError) + return } zipFileList = append(zipFileList, zipFileName) } diff --git a/taskhandler/grabissue.go b/taskhandler/grabissue.go index e729d3c..7783189 100644 --- a/taskhandler/grabissue.go +++ b/taskhandler/grabissue.go @@ -228,21 +228,22 @@ func GetOrgRepos(accToken, org string, page int64) { //GetIssueList get the repository issue list func GetIssueList(accToken, owner, repo, state string, page int) (issueList []models.HookIssue, err error) { - resp, err := http.Get(fmt.Sprintf(GiteRepoIssuesURL, owner, repo,accToken, state, page, perPage)) + giteUrl := fmt.Sprintf(GiteRepoIssuesURL, owner, repo,accToken, state, page, perPage) + resp, err := http.Get(giteUrl) if err != nil { - logs.Error("Get, GiteRepoIssuesURL: ", GiteRepoIssuesURL, ", repo: ", repo, ", err: ", err) + logs.Error("Get, GiteRepoIssuesURL: ", giteUrl, ", repo: ", repo, ", err: ", err) return issueList, err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - logs.Error("ReadAll, GiteRepoIssuesURL: ", GiteRepoIssuesURL, ", repo: ", repo, ", err: ", err) + logs.Error("ReadAll, GiteRepoIssuesURL: ", giteUrl, ", repo: ", repo, ", err: ", err) return issueList, err } //logs.Info("issue list: ", string(body)) err = json.Unmarshal(body, &issueList) if err != nil { - logs.Error("Unmarshal, GiteRepoIssuesURL: ", GiteRepoIssuesURL, ", repo: ", repo, ", err: ", err) + logs.Error("Unmarshal, GiteRepoIssuesURL: ", giteUrl, ", repo: ", repo, ", err: ", err) } return } @@ -255,19 +256,19 @@ func GetRepoDescription(repo string) (desc string) { url := fmt.Sprintf(RepoInfoURL, repo) resp, err := http.Get(url) if err != nil { - logs.Error("Get, RepoInfoURL: ", RepoInfoURL, ",err: ", err) + logs.Error("Get, RepoInfoURL: ", url, ",err: ", err) return "" } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { - logs.Error("ReadAll, RepoInfoURL: ", RepoInfoURL, ",err: ", err) + logs.Error("ReadAll, RepoInfoURL: ", url, ",err: ", err) return "" } var pkg PackageInfo err = json.Unmarshal(body, &pkg) if err != nil { - logs.Error("Unmarshal, RepoInfoURL: ", RepoInfoURL, ",err: ", err) + logs.Error("Unmarshal, RepoInfoURL: ", url, ",err: ", err) return "" } if pkg.Code == "2001" { diff --git a/taskhandler/oricvecheck.go b/taskhandler/oricvecheck.go index c072d93..a4c41ec 100644 --- a/taskhandler/oricvecheck.go +++ b/taskhandler/oricvecheck.go @@ -61,6 +61,42 @@ func CheckCveOriginData(prcnum int) (string, error) { return "", nil } -func UnLockTableData() { - +func UpdateAbnCveStatus(prcNum, days int, cveSt string) (string, error) { + beforeTime := common.GetBeforeTime(days) + cveId := int64(0) + for { + cveData, err := models.QueryAbnCve(cveSt, beforeTime, prcNum, cveId) + if err == nil && len(cveData) > 0 { + for _, cd := range cveData { + cveId = cd.CveId + issueTmp := models.IssueTemplate{CveNum: cd.CveNum, CveId: cd.CveId, OwnedComponent: cd.PackName} + err := models.GetIssueTemplateByColName(&issueTmp, "CveNum", "CveId", "OwnedComponent") + if issueTmp.TemplateId < 1{ + cd.Status = 0 + update := models.UpdateVulnCenter(&cd, "Status") + if !update { + logs.Error("update vulnCenter status fail ", cd, ", err: ", err) + continue + } + } else { + if issueTmp.IssueId < 1 || issueTmp.IssueNum == "" || len(issueTmp.IssueNum) < 2 { + _, oks := models.QueryIssueCveByNum(cd.CveNum, cd.PackName) + if oks { + continue + } + models.DeleteIssueTemplate(issueTmp.TemplateId) + cd.Status = 0 + update := models.UpdateVulnCenter(&cd, "Status") + if !update { + logs.Error("update vulnCenter status fail ", cd, ", err: ", err) + continue + } + } + } + } + } else { + break + } + } + return "", nil } -- Gitee