From eeb09e95b0091eec2dfe9210889d132b01d98c1f Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Thu, 28 Jan 2021 17:59:06 +0800 Subject: [PATCH] CVE data submitted in batches manually can be submitted normally for incomplete nvd data and blocked cve --- conf/app.conf | 4 ++-- controllers/file.go | 4 ++-- cve-py/controller/timertaskcontroller.py | 2 +- cve-py/main.py | 5 ++++- cve-py/tabletask/crawltask.py | 8 ++++++-- cve-py/tabletask/runtask.py | 6 +++--- cve-py/tabletask/spec_error_task.py | 2 +- task/issuetask.go | 15 +++++++++++++-- taskhandler/cve.go | 10 ++++++++-- taskhandler/excel.go | 8 ++++++++ 10 files changed, 48 insertions(+), 16 deletions(-) diff --git a/conf/app.conf b/conf/app.conf index fc9d4a6..b9305fd 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -76,8 +76,8 @@ issuestatisticflag = 2 issuestatistic = 0 43 14 * * * cvecredflag = 2 cvecredit = 0 53 14 * * * -urgenttaskflag = 1 -urgenttask = 0 0 9-20 * * * +urgenttaskflag = 2 +urgenttask = 0 50 9-20 * * * [gitee] #owner = cve-test diff --git a/controllers/file.go b/controllers/file.go index 78a9cf5..81af0e4 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -265,11 +265,11 @@ func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, d readErr := taskhandler.ReadWriteFile(indexFilePath, totalSlice) logs.Info(readErr) } else { - readErr :=taskhandler.ReadWriteFile(indexFilePath, subFileSlice) + readErr := taskhandler.ReadWriteFile(indexFilePath, subFileSlice) logs.Info(readErr) } taskhandler.PostFile(indexFilePath, uploadPath) - readErr :=taskhandler.ReadWriteFile(updateFilePath, subFileSlice) + readErr := taskhandler.ReadWriteFile(updateFilePath, subFileSlice) if readErr != nil { logs.Error(updateFilePath, readErr) } diff --git a/cve-py/controller/timertaskcontroller.py b/cve-py/controller/timertaskcontroller.py index 3577f9b..16bef75 100644 --- a/cve-py/controller/timertaskcontroller.py +++ b/cve-py/controller/timertaskcontroller.py @@ -28,7 +28,7 @@ def timertask(): # Warehouse, organization, and owner correspondence timing task scheduler.add_job(taskcontroller.gwcontroller, 'cron', day_of_week='0-6', hour=0, minute=2) # Manual CVE data entry timing task - scheduler.add_job(taskcontroller.runtabletask, 'cron', hour='*', minute='*/20') + scheduler.add_job(taskcontroller.runtabletask, 'cron', hour='*', minute='*/15') # Error data export and email sending timing task scheduler.add_job(taskcontroller.toexcelcontroller, 'cron', day_of_week='0-6', hour=7, minute=30) # Package name correspondence relationship timing task diff --git a/cve-py/main.py b/cve-py/main.py index f988f87..dea835e 100644 --- a/cve-py/main.py +++ b/cve-py/main.py @@ -16,7 +16,10 @@ Authors: xiaojianghui Date: 10/22/2020 11:01 AM """ from controller import timertaskcontroller +from tabletask import runtask if __name__ == '__main__': - print("The program starts, waiting for the timing task to execute") + print("Excel for emergency cve...") + runtask.handle_data() + print("Start a scheduled task...") timertaskcontroller.timertask() diff --git a/cve-py/tabletask/crawltask.py b/cve-py/tabletask/crawltask.py index 3a03d08..c289f11 100644 --- a/cve-py/tabletask/crawltask.py +++ b/cve-py/tabletask/crawltask.py @@ -20,7 +20,7 @@ from datetime import datetime def crawling(url): """ - Grab cve specific information + Grab cve specific information :param url: string :return xpth_list: list """ @@ -60,6 +60,9 @@ def crawling(url): integrity = availability = authentication = None print("No data on this vulnerability link, ", url) score_type = "" + cve_desc = str(html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')[0]) + if cve_desc: + score_type = "v3.0" else: score_type = "v2.0" element = html.xpath('//*[@id="nistV2MetricHidden"]/@value') @@ -89,7 +92,8 @@ def crawling(url): nvd_score = cve_level = cve_desc = repair_time = vector_value = attack_vector = \ access_vector = attack_complexity = access_complexity = \ privilege_required = user_interaction = scope = confidentiality = integrity = \ - availability = authentication = score_type = None + availability = authentication = None + score_type = "v3.0" print("This vulnerability link not found, ", url) else: score_type = "v3.0" diff --git a/cve-py/tabletask/runtask.py b/cve-py/tabletask/runtask.py index 412159a..b926cf0 100644 --- a/cve-py/tabletask/runtask.py +++ b/cve-py/tabletask/runtask.py @@ -23,9 +23,9 @@ import hashlib def handle_data(): """ - CVSS official website data crawling data storage database + CVSS official website data crawling data storage database """ - downloadfiletask.handle_one() + # downloadfiletask.handle_one() path = "./newexcels" if not os.path.isdir(path): print("There is currently no excel data executable") @@ -73,7 +73,7 @@ def handle_data(): crawl_list = crawltask.crawling(urls[i]) print(crawl_list) # Determine whether the database content is the latest data - if crawl_list[0]: + if crawl_list[0] or crawl_list[2]: if str(result_dict["nvd_score"]) == str(crawl_list[0]) \ and str(result_dict["vector_value"]) == str(crawl_list[4]): if result_dict['cve_status'] in [3, 4, 7]: diff --git a/cve-py/tabletask/spec_error_task.py b/cve-py/tabletask/spec_error_task.py index d86e03c..9abea93 100644 --- a/cve-py/tabletask/spec_error_task.py +++ b/cve-py/tabletask/spec_error_task.py @@ -74,7 +74,7 @@ def parse_error_excels(file_name): continue if table_one.cell(i, 4).value: cve_desc_list.append(table_one.cell(i, 4).value) - else : + else: cve_desc_list.append("") if table_one.cell(i, 2).value: cve_owner_list.append(table_one.cell(i, 2).value) diff --git a/task/issuetask.go b/task/issuetask.go index 2c4a7ac..e016339 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -499,9 +499,20 @@ func ProcIssue(issueValue models.VulnCenter, brandArry = append(brandArry, branch) } } + legalFlag := false + if issueValue.DataSource == 3 { + if len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + legalFlag = true + } + } else { + if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + legalFlag = true + } + } // Judgment of necessary fields - if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || - len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + if legalFlag { logs.Error("Field is empty: NVDScore: ", sc.NVDScore, ",Description: ", issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 18c8549..6d4ed78 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -40,15 +40,21 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNu CveRes.CveUrl = cveRef + cveData.CveNum CveRes.CveLevel = cveData.CveLevel CveRes.OrganizationID = organizationID - //CveRes.DataSource = 3 + if CveRes.DataSource == 1 { + CveRes.DataSource = 3 + } if CveRes.DataSource != 4 && CveRes.Status != 0 && CveRes.Status != 1 { CveRes.Status = 1 CveRes.IsExport = 0 } - if CveRes.DataSource == 4 && CveRes.Status != 0 && CveRes.Status != 1 { + if (CveRes.DataSource == 4 || CveRes.DataSource == 3) && CveRes.Status != 0 && CveRes.Status != 1 { issueTmp := models.IssueTemplate{CveId: CveRes.CveId, OwnedComponent: CveRes.PackName, OwnedVersion: CveRes.CveVersion} err := models.GetIssueTemplateByColName(&issueTmp, "CveId", "OwnedComponent", "OwnedVersion") if err != nil { + if CveRes.DataSource == 3 { + CveRes.Status = 0 + CveRes.IsExport = 0 + } logs.Error(err) } else { if issueTmp.Status < 3 { diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 31ef5f8..bfd875e 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -687,6 +687,14 @@ func UpdateUseOpenEulerSaNum() { sax.UpdateTime = common.GetCurTime() models.UpdateSaNumber(&sax, "status") } + var saFile models.SaFileList + saFile.FileName = "cvrf-" + use + ".xml" + models.GetCvrfFileName(&saFile, "FileName") + if saFile.FileId == 0 { + saFile.Status = 1 + saFile.CreateTime = common.GetCurTime() + models.InsertCvrfFileName(&saFile) + } } } } -- Gitee