diff --git a/conf/app.conf b/conf/app.conf index 2e28a749027155ffbeee5f18856efd6ac67adf12..d8f7ad733a94d61f46c353f9e9aae93839c6d727 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -61,7 +61,7 @@ issueoath = * * */20 * * * genexcelflag = 2 genexcel = 0 0/10 * * * * days = -30 -prcnum = 5 +prcnum = 50 printlogflag = 1 printlog = 0 0/20 * * * * unlockflag = 2 @@ -72,7 +72,7 @@ exceptflag = 2 exceptcve = 0 3 19 * * * gensaflag = 2 gensa = 0 3 19 * * * -issuestatisticflag = 1 +issuestatisticflag = 2 issuestatistic = 0 19 10 * * * [gitee] diff --git a/cve-py/controller/taskcontroller.py b/cve-py/controller/taskcontroller.py index 959f13fa56c6f56bdef0803f89f98c2098e4606f..89fd17c7fe590cd3349f54528cd91a4f9ae61176 100644 --- a/cve-py/controller/taskcontroller.py +++ b/cve-py/controller/taskcontroller.py @@ -10,12 +10,14 @@ Control task Authors: xiaojianghui +update by: zhangjianjun Date: 10/22/2020 11:01 AM """ from tabletask import (runtask, mappingtask, toexcel, export_excel_task, import_excel_task, package_committer_task, - spec_error_task, repeattask, gauss_yaml, issue_statistics) + spec_error_task, repeattask, gauss_yaml, + issue_statistics, supplement_cve) from gitwebtask import genegroup, yamltask from emailtask import sendingtask, issue_record_email from deletetask import deletefiletask @@ -31,8 +33,8 @@ def toexcelcontroller(): Returns: return None """ - - print("Export data to generate excel, send error data feedback email task start") + print("Export data to generate excel, " + "send error data feedback email task start") toexcel.MysqlToExcel().generate_table() toexcel.MysqlToExcel().update_status() sendingtask.sending() @@ -45,12 +47,13 @@ def runtabletask(): website data, and store it in the database return None """ - - print("Analyze the manual data table, crawl the CVE official website data, and start the timing task") + print("Analyze the manual data table, " + "crawl the CVE official website data, and start the timing task") spec_error_task.add_error_details() runtask.handle_data() repeattask.repeat_task() - print("Analyze the manual data table, crawl the CVE official website data task completed") + print("Analyze the manual data table, " + "crawl the CVE official website data task completed") def runmappeingtask(): @@ -59,10 +62,11 @@ def runmappeingtask(): official website data, and store it in the database return None """ - - print("The timing task of parsing the package name mapping table starts") + print("The timing task of parsing the " + "package name mapping table starts") mappingtask.to_mysql() - print("The timing task of parsing the package name mapping table completed") + print("The timing task of parsing the " + "package name mapping table completed") def gwcontroller(): @@ -70,8 +74,8 @@ def gwcontroller(): Grab warehouse yaml data return None """ - - print("Start of crawling organization, warehouse, owners correspondence relationship timing task") + print("Start of crawling organization, warehouse, " + "owners correspondence relationship timing task") genegroup.Command().handle() yamltask.Task().download() yamltask.Task().read() @@ -115,7 +119,7 @@ def run_whiltlist_task(): def run_package_committer_task(): """ - the task of parsing package_committer excels + the task of parsing package_committer excels Returns:None """ @@ -143,3 +147,14 @@ def issue_statistics_email_task(): print('Parse the issue statistics recipient list start') issue_statistics.get_issue_excel() print('Parse the issue statistics recipient list complete') + + +def supplement_cve_task(): + """ + Complete the template information of the + issue with the data on the CVE official website + return None + """ + print("Supplemental cve information task starts") + supplement_cve.supplement_cve() + print("Supplement cve information task is over") diff --git a/cve-py/controller/timertaskcontroller.py b/cve-py/controller/timertaskcontroller.py index 1daf9f87bc5acb5726ececb208684e4853d5acb7..76e44bcb8a1b61087de7d85244eee5fbbff3fed0 100644 --- a/cve-py/controller/timertaskcontroller.py +++ b/cve-py/controller/timertaskcontroller.py @@ -45,6 +45,8 @@ def timertask(): scheduler.add_job(taskcontroller.parse_opengauss_yaml_task, 'cron', day_of_week='0-6', hour=8, minute=30) # Parse the issue statistics recipient list scheduler.add_job(taskcontroller.issue_statistics_email_task, 'cron', day_of_week='0-6', hour=5, minute=30) + # Complete the template information of the issue with the data on the CVE official website + scheduler.add_job(taskcontroller.supplement_cve_task, 'cron', day_of_week='0-6', hour=1, minute=30) scheduler.start() except SystemExit as err: print("Err:", err) diff --git a/cve-py/tabletask/repeattask.py b/cve-py/tabletask/repeattask.py index f47fd504afac33165832d6e4a4eba0ba2c0977a9..7a6119722dc0f97570dad863f0cfd61e421cac12 100644 --- a/cve-py/tabletask/repeattask.py +++ b/cve-py/tabletask/repeattask.py @@ -36,7 +36,8 @@ def repeat_task(): """ print("CVE官网数据为空的CVE进行再次抓取") mysql = Mysql() - sql = "SELECT cve_num FROM cve_origin_excel WHERE score_type = %s " \ + sql = "SELECT cve_num,pack_name,cve_version " \ + "FROM cve_origin_excel WHERE score_type = %s " \ "OR score_type IS NULL OR cve_desc IS NULL OR nvd_score IS NULL" val = ('',) result = mysql.getMany(sql, val) @@ -49,10 +50,13 @@ def repeat_task(): "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ "availability=%s, authentication=%s, cve_status=%s, update_time=%s, " \ - "score_type=%s where cve_num=%s" - val = (crawllist[0], crawllist[1], crawllist[2], crawllist[3], crawllist[4], crawllist[5], crawllist[6], - crawllist[7], crawllist[8], crawllist[9], crawllist[10], crawllist[11], crawllist[12], crawllist[13], - crawllist[14], crawllist[15], 1, cur_date(), crawllist[16], i["cve_num"]) + "score_type=%s where cve_num=%s and pack_name = %s and cve_version = %s" + val = (crawllist[0], crawllist[1], crawllist[2], crawllist[3], + crawllist[4], crawllist[5], crawllist[6], + crawllist[7], crawllist[8], crawllist[9], crawllist[10], + crawllist[11], crawllist[12], crawllist[13], + crawllist[14], crawllist[15], 1, cur_date(), crawllist[16], + i["cve_num"], i["pack_name"], i["cve_version"]) mysql.update(sql, val) mysql.dispose() mysql.close() diff --git a/cve-py/tabletask/runtask.py b/cve-py/tabletask/runtask.py index 61451ab2317c77f52c42069f821f8c6bae1e0bad..412159afc633a5538b07f16a75a8f59b7a232ccc 100644 --- a/cve-py/tabletask/runtask.py +++ b/cve-py/tabletask/runtask.py @@ -59,10 +59,13 @@ def handle_data(): pack_name_list = exceltask.crawl_packname(fileName) for i in range(0, len(urls)): cve_num = str(cve_num_list[i]).strip() - print(fileName, cve_num) + cve_version = str(cve_version_list[i]) + pack_name = str(pack_name_list[i]) + print(fileName, cve_num, pack_name, cve_version) # Database query results - sql = "select * from cve_origin_excel where cve_num= %s" - val = (cve_num,) + sql = "select * from cve_origin_excel where " \ + "cve_num= %s and pack_name = %s and cve_version = %s" + val = (cve_num, pack_name, cve_version) result_dict = mysql.getOne(sql, val) # Determine whether CVE exists in the database if result_dict: @@ -71,8 +74,8 @@ def handle_data(): print(crawl_list) # Determine whether the database content is the latest data if crawl_list[0]: - if str(result_dict["nvd_score"]) == str(crawl_list[0]) and str(result_dict["vector_value"] - ) == str(crawl_list[4]): + if str(result_dict["nvd_score"]) == str(crawl_list[0]) \ + and str(result_dict["vector_value"]) == str(crawl_list[4]): if result_dict['cve_status'] in [3, 4, 7]: print("update data:" + cve_num) update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) @@ -82,13 +85,13 @@ def handle_data(): "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ "availability=%s, authentication=%s, cve_status=%s, update_time=%s " \ - "where cve_num=%s" + "where cve_num=%s and pack_name = %s and cve_version = %s" val = ( crawl_list[0], crawl_list[1], crawl_list[2], crawl_list[3], crawl_list[4], crawl_list[5], crawl_list[6], crawl_list[7], crawl_list[8], crawl_list[9], crawl_list[10], crawl_list[11], crawl_list[12], crawl_list[13], crawl_list[14], - crawl_list[15], 1, update_time, cve_num) + crawl_list[15], 1, update_time, cve_num, pack_name, cve_version) mysql.update(sql, val) mysql.dispose() except IndexError as e: @@ -105,12 +108,14 @@ def handle_data(): "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ "availability=%s, authentication=%s, cve_status=%s, update_time=%s, " \ - "score_type=%s where cve_num=%s" + "score_type=%s " \ + "where cve_num=%s and pack_name = %s and cve_version = %s" val = ( crawl_list[0], crawl_list[1], crawl_list[2], crawl_list[3], crawl_list[4], crawl_list[5], crawl_list[6], crawl_list[7], crawl_list[8], crawl_list[9], crawl_list[10], crawl_list[11], crawl_list[12], crawl_list[13], crawl_list[14], - crawl_list[15], 1, update_time, crawl_list[16], cve_num) + crawl_list[15], 1, update_time, crawl_list[16], + cve_num, pack_name, cve_version) mysql.update(sql, val) mysql.dispose() except IndexError as e: @@ -126,8 +131,6 @@ def handle_data(): cve_status = 0 try: cve_url = str(urls[i]) - cve_version = str(cve_version_list[i]) - pack_name = str(pack_name_list[i]) listx = crawltask.crawling(cve_url) sql = "INSERT INTO cve_origin_excel (cve_num, cve_url, cve_version, pack_name, score_type, " \ "nvd_score, cve_level, cve_desc, repair_time, vector_value, attack_vector, " \ diff --git a/cve-py/tabletask/supplement_cve.py b/cve-py/tabletask/supplement_cve.py new file mode 100644 index 0000000000000000000000000000000000000000..aa681acf7c8b967368898e7ad56dc427ac0130ac --- /dev/null +++ b/cve-py/tabletask/supplement_cve.py @@ -0,0 +1,201 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2021 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Supplemental cve information + +Authors: zhangjianjun +Date: 1/19/2021 11:01 AM +""" + +from tabletask import crawltask +from dbConnecttion.MysqlConn import Mysql +import time +import datetime + + +def query_cve_data(mysql): + """ + Find cve with missing necessary fields + """ + before_date = (datetime.date.today() - + datetime.timedelta(days=90)).strftime("%Y-%m-%d %H:%M:%S") + score_sql = "select cve_id,cve_num from cve_score " \ + "where nvd_score = 0 and create_time >= %s" + val = (before_date,) + cve_list = [] + score_result = mysql.getMany(score_sql, val) + if score_result and len(score_result) > 0: + for sc in score_result: + center_sql = "select cve_num, pack_name, cve_version from " \ + "cve_vuln_center where cve_id = %s and cve_num = %s" + center_val = (sc["cve_id"], sc["cve_num"]) + center_result = mysql.getOne(center_sql, center_val) + if center_result: + cve_list.append(center_result) + cve_desc_sql = "select cve_num, pack_name, cve_version from " \ + "cve_vuln_center where cve_desc = %s and create_time >= %s" + center_val = ("", before_date) + center_result = mysql.getMany(cve_desc_sql, center_val) + if center_result and len(center_result) > 0: + for ce in center_result: + cve_list.append(ce) + print(cve_list) + return cve_list + + +def update_cve(url, result_dict, cve, mysql): + """ + update data + """ + cve_num = str(cve["cve_num"]).strip() + cve_version = str(cve["cve_version"]).strip() + pack_name = str(cve["pack_name"]).strip() + # Crawler web data + crawl_list = crawltask.crawling(url) + print(crawl_list) + # Determine whether the database content is the latest data + if crawl_list[0]: + if str(result_dict["nvd_score"]) == str(crawl_list[0]) \ + and str(result_dict["vector_value"]) == str(crawl_list[4]): + if result_dict['cve_status'] in [3, 4, 7]: + print("update data:" + cve_num) + update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + try: + sql = "update cve_origin_excel set nvd_score=%s, cve_level=%s, cve_desc=%s, " \ + "repair_time=%s, vector_value=%s, attack_vector=%s, access_vector=%s, " \ + "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ + "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ + "availability=%s, authentication=%s, cve_status=%s, update_time=%s " \ + "where cve_num=%s and pack_name = %s and cve_version = %s" + val = ( + crawl_list[0], crawl_list[1], crawl_list[2], crawl_list[3], crawl_list[4], + crawl_list[5], + crawl_list[6], crawl_list[7], crawl_list[8], crawl_list[9], + crawl_list[10], crawl_list[11], crawl_list[12], crawl_list[13], crawl_list[14], + crawl_list[15], 1, update_time, cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + except IndexError as e: + mysql.dispose(0) + print("Subscript out of bounds", e) + else: + print("The database is the latest data:" + cve_num) + else: + print("update data:" + cve_num) + update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + try: + sql = "update cve_origin_excel set nvd_score=%s, cve_level=%s, cve_desc=%s, " \ + "repair_time=%s, vector_value=%s, attack_vector=%s, access_vector=%s, " \ + "attack_complexity=%s, access_complexity=%s, privilege_required=%s, " \ + "user_interaction=%s, scope=%s, confidentiality=%s, integrity=%s, " \ + "availability=%s, authentication=%s, cve_status=%s, update_time=%s, " \ + "score_type=%s " \ + "where cve_num=%s and pack_name = %s and cve_version = %s" + val = ( + crawl_list[0], crawl_list[1], crawl_list[2], crawl_list[3], crawl_list[4], + crawl_list[5], crawl_list[6], crawl_list[7], crawl_list[8], crawl_list[9], + crawl_list[10], crawl_list[11], crawl_list[12], crawl_list[13], crawl_list[14], + crawl_list[15], 1, update_time, crawl_list[16], + cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + except IndexError as e: + print("Subscript out of bounds", e) + mysql.dispose(0) + else: + print("error: ", result_dict) + + +def insert_cve(url, cve, mysql): + """ + insert data + """ + cve_num = str(cve["cve_num"]).strip() + cve_version = str(cve["cve_version"]).strip() + pack_name = str(cve["pack_name"]).strip() + create_time = update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + delete_time = None + # State 0 means new, 1 means modified + cve_status = 1 + try: + listx = crawltask.crawling(url) + if listx[2] is None or \ + len(listx[2]) < 2 or \ + listx[0] is None or \ + listx[0] == 0: + print("The data does not exist and will not be processed temporarily", listx) + return + sql = "INSERT INTO cve_origin_excel (cve_num, cve_url, cve_version, pack_name, score_type, " \ + "nvd_score, cve_level, cve_desc, repair_time, vector_value, attack_vector, " \ + "access_vector, attack_complexity, access_complexity, privilege_required, " \ + "user_interaction, scope, confidentiality, integrity, availability, " \ + "authentication, cve_status, " \ + "create_time, update_time, delete_time) " \ + "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, " \ + "%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)" + val = ( + cve_num, url, cve_version, pack_name, listx[16], listx[0], listx[1], listx[2], listx[3], + listx[4], + listx[5], listx[6], listx[7], listx[8], listx[9], listx[10], listx[11], listx[12], + listx[13], + listx[14], listx[15], cve_status, create_time, update_time, delete_time) + mysql.insertOne(sql, val) + mysql.dispose() + except IndexError as e: + print("Subscript out of bounds", e) + mysql.dispose(0) + + +def supplement_cve(): + """ + 1. Find cve with missing necessary fields; + 2. Go to the CVE official website to find the CVE information; + 3. Fill in the corresponding table again; + return None + """ + mysql = Mysql() + cve_list = query_cve_data(mysql) + if cve_list is not None and len(cve_list) > 0: + for cve in cve_list: + cve_num = str(cve["cve_num"]).strip() + cve_version = str(cve["cve_version"]).strip() + pack_name = str(cve["pack_name"]).strip() + url = "https://nvd.nist.gov/vuln/detail/" + cve_num + sql = "select * from cve_origin_excel where " \ + "cve_num= %s and pack_name = %s and cve_version = %s" + val = (cve_num, pack_name, cve_version) + result_dict = mysql.getOne(sql, val) + # Determine whether CVE exists in the database + if result_dict: + if result_dict["cve_desc"] is None or \ + len(result_dict["cve_desc"]) < 2 or \ + result_dict["nvd_score"] is None or \ + result_dict["nvd_score"] == 0: + pass + # update_cve(url, result_dict, cve, mysql) + else: + sql = "update cve_origin_excel set cve_status = %s " \ + "where cve_num= %s and pack_name = %s and cve_version = %s" + val = (1, cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + else: + insert_cve(url, cve, mysql) + sql = "select * from cve_spec_error where cve_num = %s and " \ + "cve_owner = %s and pack_name = %s" + val = (cve_num, "src-openEuler", pack_name) + result_spec_error = mysql.getOne(sql, val) + if result_spec_error: + print("过滤,修改status为6：{}".format(cve_num)) + sql = "update cve_origin_excel set cve_desc = %s, cve_status = %s " \ + "where cve_num= %s and pack_name = %s and cve_version = %s" + val = (result_spec_error["cve_desc"], 6, cve_num, pack_name, cve_version) + mysql.update(sql, val) + mysql.dispose() + mysql.close() diff --git a/models/modeldb.go b/models/modeldb.go index 36a8ce5ed9859b7988fe1fe494146fa239d6b2fa..b385c46418451fc675091e2b58eb5f8b5e987bfd 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -492,7 +492,7 @@ type OriginExcel struct { CveNum string `orm:"size(256);column(cve_num)" description:"cve编号"` CveUrl string `orm:"size(2048);column(cve_url);null" description:"cve下载链接"` CveVersion string `orm:"size(256);column(cve_version);null" description:"漏洞归属版本"` - PackName string `orm:"size(128);column(pack_name);null" description:"包名"` + PackName string `orm:"size(256);column(pack_name);null" description:"包名"` ScoreType string `orm:"size(16);column(score_type);null" description:"评分类型：v3"` NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score);null" description:"nvd评分"` CveLevel string `orm:"size(32);column(cve_level);null" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 1eb92894a0228c19078ab2ba13ee5e5aa97125ea..e8f7e117bf669c7404387d890e57458251cc5c30 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -45,6 +45,18 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNu CveRes.Status = 1 CveRes.IsExport = 0 } + if CveRes.DataSource == 4 && CveRes.Status != 0 && CveRes.Status != 1 { + issueTmp := models.IssueTemplate{CveId: CveRes.CveId, OwnedComponent: CveRes.PackName, OwnedVersion: CveRes.CveVersion} + err := models.GetIssueTemplateByColName(&issueTmp, "CveId", "OwnedComponent", "OwnedVersion") + if err != nil { + logs.Error(err) + } else { + if issueTmp.Status < 3 { + CveRes.Status = 1 + CveRes.IsExport = 0 + } + } + } //CveRes.Status = 1 openEusa, operr := models.QueryOpenEulerSAByCveId(CveRes.CveId) if operr == false {