diff --git a/.gitignore b/.gitignore index bac04964914eb0747b738f04849c819b8195d8da..c42b1d66a498aeb9310a788e457ca81a7254d113 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ __pycache__ *.dll *.so *.dylib - +*.pyc # Test binary, built with `go test -c` *.test diff --git a/cve-py/controller/taskcontroller.py b/cve-py/controller/taskcontroller.py index 1c710cb54118fcfb2ac01b9912b5914f926fc0fc..d7d40605ac8c3d40c59f72fbd055631ba02e8706 100644 --- a/cve-py/controller/taskcontroller.py +++ b/cve-py/controller/taskcontroller.py @@ -13,9 +13,9 @@ Authors: xiaojianghui Date: 10/22/2020 11:01 AM """ -from tabletask import runtask, mappingtask, toexcel +from tabletask import runtask, mappingtask, toexcel, export_excel_task, import_excel_task from gitwebtask import genegroup, yamltask -from emailtask import sendingtask +from emailtask import sendingtask, issue_record_email from downloadtask import downloadfiletask from deletetask import deletefiletask @@ -83,6 +83,7 @@ def rundownloadtask(): print("Download file timing task starts") downloadfiletask.handle_one() downloadfiletask.handle_two() + downloadfiletask.handle_three() print("Download file task completed") @@ -95,3 +96,25 @@ def rundelfiletask(): print("Clean up the folder timed task starts") deletefiletask.deldir("./oldexcels") print("Clean up the folder task completed") + + +def run_issue_record_task(): + """ + issues record email task + return None + """ + print('issues record email task start') + export_excel_task.generate_excels(1, './export_excels/create_issues') + export_excel_task.generate_excels(2, './export_excels/untreated_issues') + issue_record_email.sending() + print('issues record email task complete') + + +def run_whiltlist_task(): + """ + the task of parsing package whitelist + return None + """ + print('the task of parsing package whitelist start') + import_excel_task.import_data() + print('the task of parsing package whitelist complete') diff --git a/cve-py/controller/timertaskcontroller.py b/cve-py/controller/timertaskcontroller.py index 85595907b144d20ed38acf86fae08e9b8b9f3e0f..6a03e804769974352d968ff21b658fb761367b3e 100644 --- a/cve-py/controller/timertaskcontroller.py +++ b/cve-py/controller/timertaskcontroller.py @@ -30,14 +30,17 @@ def timertask(): # Manual CVE data entry timing task scheduler.add_job(taskcontroller.runtabletask, 'cron', day_of_week='0-6', hour=0, minute=30) # Error data export and email sending timing task - scheduler.add_job(taskcontroller.toexcelcontroller, 'cron', day_of_week='0-6', hour=7, minute=30) + scheduler.add_job(taskcontroller.toexcelcontroller, 'interval', hours=2) # Package name correspondence relationship timing task scheduler.add_job(taskcontroller.runmappeingtask, 'cron', day_of_week='0-6', hour=7, minute=30) # Download files timing task - scheduler.add_job(taskcontroller.rundownloadtask, 'cron', day_of_week='0-6', hour=0, minute=1) + scheduler.add_job(taskcontroller.rundownloadtask, 'interval', hours=1) # Delete ,iles timed tasks that are more than one month old scheduler.add_job(taskcontroller.rundelfiletask, 'cron', day_of_week='0-6', hour=9, minute=30) + # issue record email feedback + scheduler.add_job(taskcontroller.run_issue_record_task, 'cron', day_of_week='0-6', hour=7, minute=30) + # analysis the table package whitelist + scheduler.add_job(taskcontroller.run_whiltlist_task, 'interval', hours=2) scheduler.start() except SystemExit as err: print("Err:", err) - diff --git a/cve-py/downloadtask/downloadfiletask.py b/cve-py/downloadtask/downloadfiletask.py index 5d7969c3a072227c887a5b1c2345cd9114e7f3d0..1a83a8ee6f939eadeea4bec453d25acbe4f9e77a 100644 --- a/cve-py/downloadtask/downloadfiletask.py +++ b/cve-py/downloadtask/downloadfiletask.py @@ -17,6 +17,7 @@ import requests import lxml.etree as etree import shutil import os +import datetime def handle_one(): @@ -43,15 +44,17 @@ def handle_one(): name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] except IndexError: break - if name.endswith(".xls") or name.endswith(".xlsx"): - url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/newexcels/" + name - r = requests.get(url) - print(r.status_code) - with open("./newexcels/" + name, 'wb') as code: - code.write(r.content) - if os.path.exists("./newexcels/" + name): - print("Successfully downloaded the cve manual data form:" + name) - i += 2 + if name and len(name) > 13 and name.split('.')[0][-10:] > ( + datetime.datetime.now() + datetime.timedelta(-2)): + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/newexcels/" + name + r = requests.get(url) + print(r.status_code) + with open("./newexcels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./newexcels/" + name): + print("Successfully downloaded the cve manual data form:" + name) + i += 2 def handle_two(): @@ -78,12 +81,51 @@ def handle_two(): name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] except IndexError: break - if name.endswith(".xls") or name.endswith(".xlsx"): - url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/mappingexcels/" + name - r = requests.get(url) - print(r.status_code) - with open("./mappingexcels/" + name, 'wb') as code: - code.write(r.content) - if os.path.exists("./mappingexcels/" + name): - print("Download the package name mapping table successfully:" + name) - i += 2 + if name and len(name) > 13 and name.split('.')[0][-10:] > ( + datetime.datetime.now() + datetime.timedelta(-2)): + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/mappingexcels/" + name + r = requests.get(url) + print(r.status_code) + with open("./mappingexcels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./mappingexcels/" + name): + print("Download the package name mapping table successfully:" + name) + i += 2 + + +def handle_three(): + """ + download file + """ + if os.listdir("./import_excels"): + shutil.rmtree("./import_excels") + os.mkdir("./mappingexcels") + url = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/import_excels' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name and len(name) > 13 and name.split('.')[0][-10:] > ( + datetime.datetime.now() + datetime.timedelta(-2)): + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/import_excels/" + name + r = requests.get(url) + print(r.status_code) + with open("./import_excels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./import_excels/" + name): + print("Download the Package whitelist table successfully:" + name) + i += 2 diff --git a/cve-py/emailtask/issue_record_email.py b/cve-py/emailtask/issue_record_email.py new file mode 100644 index 0000000000000000000000000000000000000000..5cc0d8b14d4bf5032347edff48b8a8b6164791f7 --- /dev/null +++ b/cve-py/emailtask/issue_record_email.py @@ -0,0 +1,37 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +send email + +Authors: xiaojianghui +Date: 11/06/2020 11:01 AM +""" + +from dbConnecttion.MysqlConn import Mysql +from emailtask import sendemail +import os + + +def sending(): + """ + Query all recipients + """ + mysql = Mysql() + sql = "select email_name from cve_email_list where email_type=1" + result = mysql.getMany(sql) + mysql.close() + print(result) + for i in result: + subject = "This time cve creates an issue record" + content = "The cve table that has created an issue and the cve table that has not created an issue" + sendemail.send_email("smtp.gmail.com", 587, + os.getenv("CVE_EMAIL_SENDADDR"), + os.getenv("CVE_EMAIL_PASSWORD"), + str(i['email_name']), './export_excels', subject, content) + for filex in os.listdir('./export_excels'): + os.remove('./export_excels/' + filex) diff --git a/cve-py/emailtask/sendemail.py b/cve-py/emailtask/sendemail.py index 7eb29207ad3c3332675a98a155ba6d85dbf12f0f..c802c9b8ce4bfc0723ac70c75de2f776958e5645 100644 --- a/cve-py/emailtask/sendemail.py +++ b/cve-py/emailtask/sendemail.py @@ -20,7 +20,7 @@ import email.mime.text import email.mime.application as application -def send_email(smtp_host, smtp_port, sendAddr, password, recipientAddrs, subject='', content=''): +def send_email(smtp_host, smtp_port, sendAddr, password, recipientAddrs, path, subject='', content=''): """ send email :param smtp_host:smpt.gmail.com @@ -40,26 +40,26 @@ def send_email(smtp_host, smtp_port, sendAddr, password, recipientAddrs, subject txt = email.mime.text.MIMEText(content, 'plain', 'utf-8') msg.attach(txt) # Add attachment address - files = os.listdir("./problemexcels") + files = os.listdir(path) if not files: - print("No error data sheet") + print("No data sheet") return for fileName in files: - try: - part = application.MIMEApplication(open('./problemexcels/' + fileName, 'rb').read()) - part.add_header('Content-Disposition', 'attachment', filename=fileName) - msg.attach(part) - smtpSSLClient = smtplib.SMTP(smtp_host, smtp_port) - smtpSSLClient.ehlo() - smtpSSLClient.starttls() - loginRes = smtpSSLClient.login(sendAddr, password) - print("Login result:loginRes=", loginRes) - if loginRes and loginRes[0] == 235: - print("login successful,code=[loginRes[0]]") - smtpSSLClient.sendmail(sendAddr, recipientAddrs, str(msg)) - print("mail has been send successfully. message: ", str(msg)) - smtpSSLClient.quit() - else: - print("login failed,code= ", loginRes[0]) - except SystemExit as e: - print("Failed to send,Exception:e= ", e) + part = application.MIMEApplication(open(path + '/' + fileName, 'rb').read()) + part.add_header('Content-Disposition', 'attachment', filename=fileName) + msg.attach(part) + try: + smtpSSLClient = smtplib.SMTP(smtp_host, smtp_port) + smtpSSLClient.ehlo() + smtpSSLClient.starttls() + loginRes = smtpSSLClient.login(sendAddr, password) + print("Login result:loginRes=", loginRes) + if loginRes and loginRes[0] == 235: + print("login successful,code=[loginRes[0]]") + smtpSSLClient.sendmail(sendAddr, recipientAddrs, str(msg)) + print("mail has been send successfully. message: ", str(msg)) + smtpSSLClient.quit() + else: + print("login failed,code= ", loginRes[0]) + except SystemExit as e: + print("Failed to send,Exception:e= ", e) diff --git a/cve-py/emailtask/sendingtask.py b/cve-py/emailtask/sendingtask.py index 3cc47f6190dd26f1a61ad51311baa3f3275d5e9a..a6dedef3ecdc6ede551217d32189a3aabe38ad7e 100644 --- a/cve-py/emailtask/sendingtask.py +++ b/cve-py/emailtask/sendingtask.py @@ -33,4 +33,4 @@ def sending(): sendemail.send_email("smtp.gmail.com", 587, os.getenv("CVE_EMAIL_SENDADDR"), os.getenv("CVE_EMAIL_PASSWORD"), - str(i['email_name']), subject, content) + str(i['email_name']), './problemexcels', subject, content) diff --git a/cve-py/export_excels/test.xls b/cve-py/export_excels/test.xls new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cve-py/import_excels/packagewhitelist_2020-11-06.xlsx b/cve-py/import_excels/packagewhitelist_2020-11-06.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..141f5efb23f416b67d378bb3a16c4645c93c95a9 Binary files /dev/null and b/cve-py/import_excels/packagewhitelist_2020-11-06.xlsx differ diff --git a/cve-py/main.py b/cve-py/main.py index 6b05b88d5e5f3e0026cd8a5afd77b8b290cce30c..c8b0f648686bce6050ce004714bbeef4b8db357a 100644 --- a/cve-py/main.py +++ b/cve-py/main.py @@ -16,11 +16,11 @@ Authors: xiaojianghui Date: 10/22/2020 11:01 AM """ -from controller import timertaskcontroller +from controller import timertaskcontroller, taskcontroller if __name__ == '__main__': - + taskcontroller.run_issue_record_task() print("The program starts, waiting for the timing task to execute") timertaskcontroller.timertask() diff --git "a/cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\273.xls" "b/cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\2732020-11-06.xls" similarity index 100% rename from "cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\273.xls" rename to "cve-py/mappingexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\2732020-11-06.xls" diff --git a/cve-py/newexcels/20.03-CVE-10-10.xlsx b/cve-py/newexcels/20.03-CVE-10-10.xlsx deleted file mode 100644 index 67c3c08c98677dac89f2f5f3e30f53f8679c1f26..0000000000000000000000000000000000000000 Binary files a/cve-py/newexcels/20.03-CVE-10-10.xlsx and /dev/null differ diff --git a/cve-py/newexcels/20.03-CVE-2020-11-04.xlsx b/cve-py/newexcels/20.03-CVE-2020-11-04.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..c1f456cd4292d535ad597f1e6b1f5bef2126d83c Binary files /dev/null and b/cve-py/newexcels/20.03-CVE-2020-11-04.xlsx differ diff --git a/cve-py/newexcels/20.09-CVE-2019.xlsx b/cve-py/newexcels/20.09-CVE-2019.xlsx deleted file mode 100644 index 5e2f6b3aff988c31130c190c63e3b9c6f563eaf3..0000000000000000000000000000000000000000 Binary files a/cve-py/newexcels/20.09-CVE-2019.xlsx and /dev/null differ diff --git a/cve-py/oldexcels/test.xls b/cve-py/oldexcels/test.xls new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git "a/cve-py/oldexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\273.xls" "b/cve-py/oldexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\273.xls" deleted file mode 100644 index ed8729216d7bd47bad4b59ee4ee1a91d8f8c6d16..0000000000000000000000000000000000000000 Binary files "a/cve-py/oldexcels/trust\345\214\205\345\220\215\346\230\240\345\260\204\345\205\263\347\263\273.xls" and /dev/null differ diff --git a/cve-py/problemexcels/error_data_2020-10-30.xls b/cve-py/problemexcels/error_data_2020-10-30.xls deleted file mode 100644 index 69873df713513d116327f113a124cb9cbefbfc1d..0000000000000000000000000000000000000000 Binary files a/cve-py/problemexcels/error_data_2020-10-30.xls and /dev/null differ diff --git a/cve-py/problemexcels/error_data_2020-11-02.xls b/cve-py/problemexcels/error_data_2020-11-02.xls deleted file mode 100644 index ff5989fe03875f5ed19168ade5e699fd57aad4e3..0000000000000000000000000000000000000000 Binary files a/cve-py/problemexcels/error_data_2020-11-02.xls and /dev/null differ diff --git a/cve-py/problemexcels/test.xls b/cve-py/problemexcels/test.xls new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/cve-py/tabletask/export_excel_task.py b/cve-py/tabletask/export_excel_task.py new file mode 100644 index 0000000000000000000000000000000000000000..f53391ca2f1e04fabe7d9fb600d2f7febb1f2659 --- /dev/null +++ b/cve-py/tabletask/export_excel_task.py @@ -0,0 +1,98 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +export data from mysql + +Authors: xiaojianghui +Date: 10/22/2020 11:01 AM +""" + +import time +import xlwt +from dbConnecttion.MysqlConn import Mysql + + +def cur_date(): + """ + current date + :return createTime: string + """ + create_time = time.strftime("%Y-%m-%d", time.localtime()) + return create_time + + +def get_results(status): + """ + Execute mysql + :param status: int + :return: list + """ + mysql = Mysql() + sql = "select * from cve_issue_create_record where status = %s" + val = (status,) + results = mysql.getMany(sql, val) + print(results) + mysql.dispose() + mysql.close() + return results + + +def update_status(status): + """ + Execute mysql + :param status: int + :return: list + """ + mysql = Mysql() + sql = 'update cve_issue_create_record set status = %s where status = %s' + val = (4, status) + mysql.update(sql, val) + mysql.dispose() + mysql.close() + + +def generate_excels(status, path): + """ + export excel + :param status, path: int, string + :return: None + """ + results = get_results(status) + if not results: + print("无issue记录") + return False + f = xlwt.Workbook() + sheet1 = f.add_sheet('sheet1', cell_overwrite_ok=True) + # Column field + column_names = ['id', 'cve_id', 'cve_num', 'cve_desc', 'cve_level', 'cve_version', 'repair_time', 'pack_name', + 'nvd_score', 'n_vector_value', 'create_time', 'update_time', 'delete_time', 'status'] + # Write the first row, column name + for i in range(0, len(column_names)): + sheet1.write(0, i, column_names[i]) + # Write multiple lines + num = 0 + for i in results: + sheet1.write(num + 1, 0, i["id"]) + sheet1.write(num + 1, 1, i["cve_id"]) + sheet1.write(num + 1, 2, i["cve_num"]) + sheet1.write(num + 1, 3, i["cve_desc"]) + sheet1.write(num + 1, 4, i["cve_level"]) + sheet1.write(num + 1, 5, i["cve_version"]) + sheet1.write(num + 1, 6, i["repair_time"]) + sheet1.write(num + 1, 7, i["pack_name"]) + sheet1.write(num + 1, 8, i["nvd_score"]) + sheet1.write(num + 1, 9, i["n_vector_value"]) + sheet1.write(num + 1, 10, i['create_time']) + sheet1.write(num + 1, 11, i["update_time"]) + sheet1.write(num + 1, 12, i["delete_time"]) + sheet1.write(num + 1, 13, i['status']) + num += 1 + # save document + f.save(path + '-' + cur_date() + '.xls') + update_status(status) diff --git a/cve-py/tabletask/import_excel_task.py b/cve-py/tabletask/import_excel_task.py new file mode 100644 index 0000000000000000000000000000000000000000..c906548bb591e3386411b1366c709fcf3aad609a --- /dev/null +++ b/cve-py/tabletask/import_excel_task.py @@ -0,0 +1,94 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +import data from excel + +Authors: xiaojianghui +Date: 11/06/2020 11:01 AM +""" + +import xlrd +import time +import os +from dbConnecttion.MysqlConn import Mysql + + +def parse_excel(file_name): + """ + parse excel, import data + parse: file_name + return: list + """ + pack_name_list = [] + version_list = [] + branchs_list = [] + status_list = [] + total_list = [] + try: + data = xlrd.open_workbook('./import_excels/' + file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + pack_name_list.append(table_one.cell(i, 0).value) + for i in range(1, row_number): + version_list.append(table_one.cell(i, 1).value) + for i in range(1, row_number): + status_list.append(table_one.cell(i, 2).value) + for i in range(1, row_number): + branchs_list.append(table_one.cell(i, 3).value) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found:Sheet1", e) + total_list.append(pack_name_list) + total_list.append(version_list) + total_list.append(status_list) + total_list.append(branchs_list) + return total_list + + +def cur_date(): + """ + current date + :return createTime: string + """ + create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + return create_time + + +def import_data(): + """ + import excel + :param filename: + :return none + """ + files = os.listdir('./import_excels') + for file_name in files: + result = parse_excel(file_name) + mysql = Mysql() + for i in range(0, len(result[0])): + sql = 'select * from cve_issue_repo_whitelist where package_name = %s and version = %s' + val = (result[0][i], result[1][i]) + mysql_data = mysql.getOne(sql, val) + if mysql_data: + print("更新数据package_name:" + result[0][i]) + sql = 'update cve_issue_repo_whitelist set status = %s, branchs = %s, update_time = %s where ' \ + 'package_name = %s and version = %s' + val = (result[2][i], result[3][i], cur_date(), result[0][i], result[1][i]) + mysql.update(sql, val) + mysql.dispose() + else: + print('插入数据') + sql = 'insert into cve_issue_repo_whitelist (package_name, version, status, branchs, create_time, ' \ + 'update_time, delete_time) values (%s, %s, %s, %s, %s, %s, %s)' + val = (result[0][i], result[1][i], result[2][i], result[3][i], cur_date(), None, None) + mysql.insertOne(sql, val) + mysql.dispose() + mysql.close() + os.remove('./import_excels/' + file_name)