diff --git a/conf/app.conf b/conf/app.conf index fc9d4a682cc8c9fb4f4f67d15ee4c160323ac589..b9305fd84fef74b2237fb9f6db0ad21031e229a7 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -76,8 +76,8 @@ issuestatisticflag = 2 issuestatistic = 0 43 14 * * * cvecredflag = 2 cvecredit = 0 53 14 * * * -urgenttaskflag = 1 -urgenttask = 0 0 9-20 * * * +urgenttaskflag = 2 +urgenttask = 0 50 9-20 * * * [gitee] #owner = cve-test diff --git a/controllers/file.go b/controllers/file.go index 78a9cf50c654f36f0500d4b7e17736e7b55e3865..81af0e43d4a12fb5794e3af08cc32c8f22d0e239 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -265,11 +265,11 @@ func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, d readErr := taskhandler.ReadWriteFile(indexFilePath, totalSlice) logs.Info(readErr) } else { - readErr :=taskhandler.ReadWriteFile(indexFilePath, subFileSlice) + readErr := taskhandler.ReadWriteFile(indexFilePath, subFileSlice) logs.Info(readErr) } taskhandler.PostFile(indexFilePath, uploadPath) - readErr :=taskhandler.ReadWriteFile(updateFilePath, subFileSlice) + readErr := taskhandler.ReadWriteFile(updateFilePath, subFileSlice) if readErr != nil { logs.Error(updateFilePath, readErr) } diff --git a/cve-py/controller/timertaskcontroller.py b/cve-py/controller/timertaskcontroller.py index 3577f9b8409cf22f69e1cc80ad0b7e42dd446772..16bef756962c52568ede21ebdef525f4256b8d31 100644 --- a/cve-py/controller/timertaskcontroller.py +++ b/cve-py/controller/timertaskcontroller.py @@ -28,7 +28,7 @@ def timertask(): # Warehouse, organization, and owner correspondence timing task scheduler.add_job(taskcontroller.gwcontroller, 'cron', day_of_week='0-6', hour=0, minute=2) # Manual CVE data entry timing task - scheduler.add_job(taskcontroller.runtabletask, 'cron', hour='*', minute='*/20') + scheduler.add_job(taskcontroller.runtabletask, 'cron', hour='*', minute='*/15') # Error data export and email sending timing task scheduler.add_job(taskcontroller.toexcelcontroller, 'cron', day_of_week='0-6', hour=7, minute=30) # Package name correspondence relationship timing task diff --git a/cve-py/main.py b/cve-py/main.py index f988f87b7beae364ba70a7f312d389af54cf89b5..dea835e93e6ce285096115acf15c1723f5365497 100644 --- a/cve-py/main.py +++ b/cve-py/main.py @@ -16,7 +16,10 @@ Authors: xiaojianghui Date: 10/22/2020 11:01 AM """ from controller import timertaskcontroller +from tabletask import runtask if __name__ == '__main__': - print("The program starts, waiting for the timing task to execute") + print("Excel for emergency cve...") + runtask.handle_data() + print("Start a scheduled task...") timertaskcontroller.timertask() diff --git a/cve-py/tabletask/crawltask.py b/cve-py/tabletask/crawltask.py index 3a03d08efc940d0259e22d1f1e25e94e868aeb7d..c289f1136c2ecf236a324a1cbb56e60c2aa21849 100644 --- a/cve-py/tabletask/crawltask.py +++ b/cve-py/tabletask/crawltask.py @@ -20,7 +20,7 @@ from datetime import datetime def crawling(url): """ - Grab cve specific information + Grab cve specific information :param url: string :return xpth_list: list """ @@ -60,6 +60,9 @@ def crawling(url): integrity = availability = authentication = None print("No data on this vulnerability link, ", url) score_type = "" + cve_desc = str(html.xpath('//*[@id="vulnDetailTableView"]/tr/td/div/div[1]/p[1]/text()')[0]) + if cve_desc: + score_type = "v3.0" else: score_type = "v2.0" element = html.xpath('//*[@id="nistV2MetricHidden"]/@value') @@ -89,7 +92,8 @@ def crawling(url): nvd_score = cve_level = cve_desc = repair_time = vector_value = attack_vector = \ access_vector = attack_complexity = access_complexity = \ privilege_required = user_interaction = scope = confidentiality = integrity = \ - availability = authentication = score_type = None + availability = authentication = None + score_type = "v3.0" print("This vulnerability link not found, ", url) else: score_type = "v3.0" diff --git a/cve-py/tabletask/runtask.py b/cve-py/tabletask/runtask.py index 412159afc633a5538b07f16a75a8f59b7a232ccc..b926cf02702358aee7cbb752d498f0fd450a0b1c 100644 --- a/cve-py/tabletask/runtask.py +++ b/cve-py/tabletask/runtask.py @@ -23,9 +23,9 @@ import hashlib def handle_data(): """ - CVSS official website data crawling data storage database + CVSS official website data crawling data storage database """ - downloadfiletask.handle_one() + # downloadfiletask.handle_one() path = "./newexcels" if not os.path.isdir(path): print("There is currently no excel data executable") @@ -73,7 +73,7 @@ def handle_data(): crawl_list = crawltask.crawling(urls[i]) print(crawl_list) # Determine whether the database content is the latest data - if crawl_list[0]: + if crawl_list[0] or crawl_list[2]: if str(result_dict["nvd_score"]) == str(crawl_list[0]) \ and str(result_dict["vector_value"]) == str(crawl_list[4]): if result_dict['cve_status'] in [3, 4, 7]: diff --git a/cve-py/tabletask/spec_error_task.py b/cve-py/tabletask/spec_error_task.py index d86e03c0d46ffff8cc939e1ecd1ba36d11ec3367..9abea93fe1843d07e23d662fb997ba1f78bc5e5c 100644 --- a/cve-py/tabletask/spec_error_task.py +++ b/cve-py/tabletask/spec_error_task.py @@ -74,7 +74,7 @@ def parse_error_excels(file_name): continue if table_one.cell(i, 4).value: cve_desc_list.append(table_one.cell(i, 4).value) - else : + else: cve_desc_list.append("") if table_one.cell(i, 2).value: cve_owner_list.append(table_one.cell(i, 2).value) diff --git a/task/issuetask.go b/task/issuetask.go index 2c4a7accce581679b7aa550eece53744d7828ff0..e016339433cc9771e60bf7e2fcc566be7fe327ef 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -499,9 +499,20 @@ func ProcIssue(issueValue models.VulnCenter, brandArry = append(brandArry, branch) } } + legalFlag := false + if issueValue.DataSource == 3 { + if len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + legalFlag = true + } + } else { + if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + legalFlag = true + } + } // Judgment of necessary fields - if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || - len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + if legalFlag { logs.Error("Field is empty: NVDScore: ", sc.NVDScore, ",Description: ", issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 18c8549fc45132f1edb79296b06e1e997a4c2c5a..6d4ed78c78252e2aa32b527b92a1cba46c906ab6 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -40,15 +40,21 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNu CveRes.CveUrl = cveRef + cveData.CveNum CveRes.CveLevel = cveData.CveLevel CveRes.OrganizationID = organizationID - //CveRes.DataSource = 3 + if CveRes.DataSource == 1 { + CveRes.DataSource = 3 + } if CveRes.DataSource != 4 && CveRes.Status != 0 && CveRes.Status != 1 { CveRes.Status = 1 CveRes.IsExport = 0 } - if CveRes.DataSource == 4 && CveRes.Status != 0 && CveRes.Status != 1 { + if (CveRes.DataSource == 4 || CveRes.DataSource == 3) && CveRes.Status != 0 && CveRes.Status != 1 { issueTmp := models.IssueTemplate{CveId: CveRes.CveId, OwnedComponent: CveRes.PackName, OwnedVersion: CveRes.CveVersion} err := models.GetIssueTemplateByColName(&issueTmp, "CveId", "OwnedComponent", "OwnedVersion") if err != nil { + if CveRes.DataSource == 3 { + CveRes.Status = 0 + CveRes.IsExport = 0 + } logs.Error(err) } else { if issueTmp.Status < 3 { diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 31ef5f8778c207610bcb943a3bf993435104c256..bfd875ed2d8e051d881a4dfd6d7a9fd5f23c6145 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -687,6 +687,14 @@ func UpdateUseOpenEulerSaNum() { sax.UpdateTime = common.GetCurTime() models.UpdateSaNumber(&sax, "status") } + var saFile models.SaFileList + saFile.FileName = "cvrf-" + use + ".xml" + models.GetCvrfFileName(&saFile, "FileName") + if saFile.FileId == 0 { + saFile.Status = 1 + saFile.CreateTime = common.GetCurTime() + models.InsertCvrfFileName(&saFile) + } } } }