diff --git a/conf/app.conf b/conf/app.conf index 9edc409fd35e41aaba02abb28d869ac708ed9b48..7a455dab6b77a5342345b2dd98b6b80ca029ed38 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -20,6 +20,7 @@ approveCmd = "/approve" closeCmd = "/close" labelFixed = "CVE/FIXED" labelUnFix = "CVE/UNFIXED" +labeUnaffected = "CVE/UNAFFECTED" fileDir = "download" # release package download url rpUrl = "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" @@ -47,13 +48,13 @@ maxsize=204800 ymalflag = 2 getymal = 0 0 1 * * 1 cveflag = 2 -getcve = 0 0 3 * * * +getcve = 0 13 19 * * * oricveflag = 2 -oricvecheck = 0 0 5 * * * +oricvecheck = 0 16 19 * * * getissueflag = 2 -getissue = 0 20 5 * * * +getissue = 0 22 19 * * * issueflag = 2 -createissue = 0 0 6 * * * +createissue = 0 35 10 * * * test = 0/10 * * * * * gittokenflag = 2 issueoath = * * */20 * * * @@ -63,6 +64,10 @@ days = -30 prcnum = 50 printlogflag = 2 printlog = 0 0/20 * * * * +unlockflag = 2 +unlock = 0 30 19 * * * +hookflag = 1 +prochook = 0 34 17 * * * [gitee] @@ -124,4 +129,13 @@ snsuffix = 1002 # Version package excel download address v_pack_20_03_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv # Time difference in different time zones -sa_timestamp_zone = 28800 \ No newline at end of file +sa_timestamp_zone = 28800 + +[xml] +updateinfo_path = download/updateinfo.xml + +[email] +email_name = "${EMAIL_NAME||***}" +email_pwd = "${EMAIL_PWD||***}" +email_host = smtp.gmail.com +email_port = 587 \ No newline at end of file diff --git a/conf/product_app.conf b/conf/product_app.conf index 8ecfb5a62560c7f24672aeda86f4f9edcc219f09..a67f2824f2393942c0f96b7b3284b7dc6c0414c1 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -20,6 +20,7 @@ approveCmd = "/approve" closeCmd = "/close" labelFixed = "CVE/FIXED" labelUnFix = "CVE/UNFIXED" +labeUnaffected = "CVE/UNAFFECTED" fileDir = "download" # release package download url rpUrl = "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" @@ -64,6 +65,10 @@ days = -30 prcnum = 50 printlogflag = 1 printlog = 0 0 1 * * 1 +unlockflag = 1 +unlock = 0 0 12 * * * +hookflag = 1 +prochook = 0 1 8 * * * [gitee] @@ -121,4 +126,13 @@ snsuffix = 1002 # Version package excel download address v_pack_20_03_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv # Time difference in different time zones -sa_timestamp_zone = 28800 \ No newline at end of file +sa_timestamp_zone = 28800 + +[xml] +updateinfo_path = download/updateinfo.xml + +[email] +email_name = "${EMAIL_NAME||***}" +email_pwd = "${EMAIL_PWD||***}" +email_host = smtp.gmail.com +email_port = 587 \ No newline at end of file diff --git a/controllers/hook.go b/controllers/hook.go index c199502a30e9a0fd8e8bb7eef018ec59bf51e74a..42e6a0b16d164863d1824e0d55eb57e0ea7bc92a 100644 --- a/controllers/hook.go +++ b/controllers/hook.go @@ -17,6 +17,7 @@ import ( "regexp" "strconv" "strings" + "sync" ) var ( @@ -71,6 +72,8 @@ const ( CommentReviewRemindMaintainer = "@%v 经过 cve-manager 解析 openEuler评分 已改变 需要等待安全组成员审核通过以后, 才能进行后续操作." ) +var comLock sync.Mutex + //HookEventControllers gitee hook callback type HookEventControllers struct { beego.Controller @@ -350,6 +353,7 @@ func closeIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTempla func handleIssueStateChange(issueHook *models.IssuePayload) error { unFix := beego.AppConfig.String("labelUnFix") fixed := beego.AppConfig.String("labelFixed") + uNaffected := beego.AppConfig.String("labeUnaffected") issueTmp := models.IssueTemplate{} if issueHook.Issue.Repository.Path != "" && len(issueHook.Issue.Repository.Path) > 1 { issueTmp.IssueNum = issueHook.Iid @@ -408,7 +412,7 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { issueTmp.Status = 4 issueTmp.IssueStatus = 6 cveCenter.IsExport = 2 - issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, unFix) + issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, uNaffected) } updateBool := updateTempAndCenter(issueTmp, cveCenter, token, owner) if !updateBool { @@ -750,10 +754,56 @@ func isNormalCloseIssue(cveID int64, issueState int8) bool { // update data func updateTempAndCenter(issueTmp models.IssueTemplate, cveCenter models.VulnCenter, token, owner string) bool { appearErr := 0 - err := models.UpdateIssueTemplate(&issueTmp, "status", "issue_status", + uNaffected := beego.AppConfig.String("labeUnaffected") + sn := models.SecurityNotice{CveId: issueTmp.CveId} + snErr := sn.Read("cve_id") + if snErr != nil { + logs.Error("err: ", snErr) + } else { + if issueTmp.Status > 2 { + affectBrandFlag := false + affectBranchsxList := []string{} + affectedBranchs := beego.AppConfig.String("cve::affected_branchs") + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } + if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { + affectProductList := strings.Split(sn.AffectProduct, "/") + for _, brands := range affectProductList { + if len(affectBranchsxList) > 0 { + for _, affectBranch := range affectBranchsxList { + if affectBranch == brands { + affectBrandFlag = true + break + } + } + } + } + } + if !affectBrandFlag { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + issueTmp.IssueLabel = uNaffected + } + } + switch issueTmp.IssueStatus { + case 2: + sn.AffectStatus = "Fixed" + case 6: + sn.AffectStatus = "UnAffected" + default: + sn.AffectStatus = "UnFixed" + } + err := sn.Update("affect_status") + if err != nil { + appearErr++ + logs.Error(err) + } + } + tpErr := models.UpdateIssueTemplate(&issueTmp, "status", "issue_status", "status_name", "issue_label", "mt_audit_flag", "sa_audit_flag") - if err != nil { - logs.Error(err) + if tpErr != nil { + logs.Error(tpErr) appearErr += 1 } update := models.UpdateVulnCenter(&cveCenter, "is_export") @@ -766,25 +816,6 @@ func updateTempAndCenter(issueTmp models.IssueTemplate, cveCenter models.VulnCen logs.Error("update gitee issue label fail ") appearErr++ } - sn := models.SecurityNotice{CveId: issueTmp.CveId} - err = sn.Read("cve_id") - if err != nil { - logs.Error("err: ", err) - return false - } - switch issueTmp.IssueStatus { - case 2: - sn.AffectStatus = "Fixed" - case 6: - sn.AffectStatus = "UnAffected" - default: - sn.AffectStatus = "UnFixed" - } - err = sn.Update("affect_status") - if err != nil { - appearErr++ - logs.Error(err) - } if appearErr > 0 { logs.Error("handle issue state hook appear error maybe some step "+ "fail, appearErr: ", appearErr, ",issuetmp: ", issueTmp) @@ -1005,9 +1036,13 @@ func handleIssueComment(payload models.CommentPayload) { } } if mtAuditFlag { + comLock.Lock() maintainerApprove(&issueTmp, cuAccount, owner, token, fixed, unfixed) + defer comLock.Unlock() } else { + comLock.Lock() securityApprove(&issueTmp, cuAccount, owner, token, fixed, unfixed) + defer comLock.Unlock() } } } else if strings.HasPrefix(cBody, cmdClose) { @@ -1041,7 +1076,9 @@ func handleIssueComment(payload models.CommentPayload) { } } else { cBody = strings.ReplaceAll(cBody, "：", ":") + comLock.Lock() analysisComment(issueNum, cuAccount, cBody, &payload) + defer comLock.Unlock() } } @@ -1053,7 +1090,7 @@ func isReviewer(path string) bool { func analysisComment(issueNum string, cuAccount string, cBody string, payload *models.CommentPayload) { issueId := payload.Issue.Id issueTmp := models.IssueTemplate{IssueNum: issueNum, IssueId: issueId} - err := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") + err := models.GetIssueTemplateByColName(&issueTmp, "IssueNum", "IssueId") if err != nil { logs.Error(err) return @@ -1063,6 +1100,13 @@ func analysisComment(issueNum string, cuAccount string, cBody string, payload *m logs.Error("The issue has been closed and cannot be operated again,issuetmp: ", issueTmp) return } + // Ignore this comment + if cuAccount != "" && len(cuAccount) > 1 { + if cuAccount == "openeuler-ci-bot" { + logs.Error("openeuler-ci-bot, Ignore this comment") + return + } + } canVerfy := false issueTmp.MtAuditFlag = 1 accessToken := os.Getenv("GITEE_TOKEN") @@ -1078,8 +1122,8 @@ func analysisComment(issueNum string, cuAccount string, cBody string, payload *m case "cve_analysis": if v != "" && len(v) > 1 { issueTmp.CveAnalysis = v + cols = append(cols, k) } - cols = append(cols, k) case "principle_analysis": issueTmp.PrincipleAnalysis = v cols = append(cols, k) @@ -1096,15 +1140,15 @@ func analysisComment(issueNum string, cuAccount string, cBody string, payload *m case "openeuler_vector": if v != "" && len(v) > 1 { issueTmp.OpenEulerVector = v + cols = append(cols, k) } - cols = append(cols, k) case "affected_version": if v != "" && len(v) > 1 { if paraAffectBrandBool(v) { issueTmp.AffectedVersion = v + cols = append(cols, k) } } - cols = append(cols, k) case "solution": issueTmp.Solution = v cols = append(cols, k) @@ -1137,6 +1181,8 @@ func analysisComment(issueNum string, cuAccount string, cBody string, payload *m } } } + // update gitee issue + commentUpdateIssue(issueTmp) } if _, ok := vMap["issue_package"]; ok { // handle comment package @@ -1145,8 +1191,6 @@ func analysisComment(issueNum string, cuAccount string, cBody string, payload *m logs.Error(err) } } - // update gitee issue - commentUpdateIssue(issueTmp) } if canVerfy { //Check whether the data is legal diff --git a/cve-py/controller/taskcontroller.py b/cve-py/controller/taskcontroller.py index d0d5d269efc9c2e16750b1a30a404b7ee872e43a..61095013e6aa23785844a66f1772e50e48a44f07 100644 --- a/cve-py/controller/taskcontroller.py +++ b/cve-py/controller/taskcontroller.py @@ -13,7 +13,8 @@ Authors: xiaojianghui Date: 10/22/2020 11:01 AM """ -from tabletask import runtask, mappingtask, toexcel, export_excel_task, import_excel_task, package_committer_task +from tabletask import (runtask, mappingtask, toexcel, export_excel_task, import_excel_task, package_committer_task, + spec_error_task) from gitwebtask import genegroup, yamltask from emailtask import sendingtask, issue_record_email from deletetask import deletefiletask @@ -45,6 +46,7 @@ def runtabletask(): """ print("Analyze the manual data table, crawl the CVE official website data, and start the timing task") + spec_error_task.add_error_details() runtask.handle_data() print("Analyze the manual data table, crawl the CVE official website data task completed") diff --git a/cve-py/dbConnecttion/Config.py b/cve-py/dbConnecttion/Config.py index b540a7ea850adf7dfe1a096c9c38f616761a4d02..94943992b865d1c3ea4aeb41692da6fac752c801 100644 --- a/cve-py/dbConnecttion/Config.py +++ b/cve-py/dbConnecttion/Config.py @@ -21,4 +21,3 @@ DBUSER = "cve" DBPWD = os.getenv("DB_PWDPY") DBNAME = "cvevulner" DBCHAR = "utf8" - diff --git a/cve-py/dbConnecttion/MysqlConn.py b/cve-py/dbConnecttion/MysqlConn.py index 5ef514897da4ce5329d440c51e9cb1bb5375dcf2..b346c49519a1b64408c261d4084a26e15aa7fe2b 100644 --- a/cve-py/dbConnecttion/MysqlConn.py +++ b/cve-py/dbConnecttion/MysqlConn.py @@ -21,7 +21,7 @@ Date: 10/22/2020 11:01 AM import pymysql from pymysql import cursors from DBUtils import PooledDB -from dbConnecttion import Config +from dbConnecttion import config class Mysql(object): @@ -45,11 +45,11 @@ class Mysql(object): """ if Mysql.__pool is None: __pool = PooledDB.PooledDB(creator=pymysql, mincached=1, - maxcached=20, host=Config.DBHOST, - port=Config.DBPORT, - user=Config.DBUSER, - passwd=Config.DBPWD, db=Config.DBNAME, - use_unicode="UTF-8", charset=Config.DBCHAR, + maxcached=20, host=config.DBHOST, + port=config.DBPORT, + user=config.DBUSER, + passwd=config.DBPWD, db=config.DBNAME, + use_unicode="UTF-8", charset=config.DBCHAR, cursorclass=cursors.DictCursor) return __pool.connection() return None diff --git a/cve-py/downloadtask/downloadfiletask.py b/cve-py/downloadtask/downloadfiletask.py index c91c82de7d9e807265ccb9d43114febfe54af162..02e64ac4f701d9357fdd3625b0f28e18323a2678 100644 --- a/cve-py/downloadtask/downloadfiletask.py +++ b/cve-py/downloadtask/downloadfiletask.py @@ -158,3 +158,37 @@ def handle_four(): if os.path.exists("./package_committer_excels/" + name): print("Download the package_committer table successfully:" + name) i += 2 + + +def download_spec_error_excels(): + """ + download spec_error_excels + """ + if not os.path.exists('./spec_error_excels'): + os.makedirs('./spec_error_excels') + url = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/spec_error_excels' + try: + r = requests.get(url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name.endswith(".xls") or name.endswith(".xlsx"): + url = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/spec_error_excels/" + name + r = requests.get(url) + print(r.status_code) + with open("./spec_error_excels/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists("./spec_error_excels/" + name): + print("Download the package_committer table successfully:" + name) + i += 2 diff --git a/cve-py/gitwebtask/genegroup.py b/cve-py/gitwebtask/genegroup.py index e8c499f54a0f926fdb5675db27717cdb78cf5ffd..8b91186253240581090ebc92764c3768be011e66 100644 --- a/cve-py/gitwebtask/genegroup.py +++ b/cve-py/gitwebtask/genegroup.py @@ -57,7 +57,7 @@ class Command(object): val = (sig_name,) flag = mysql.getOne(sql, val) if not flag: - print("insert data") + print("insert data:{}".format(sig_name)) sql = "insert into cve_git_repo_groups (group_name) values (%s)" val = (sig_name,) mysql.insertOne(sql, val) diff --git a/cve-py/gitwebtask/yamltask.py b/cve-py/gitwebtask/yamltask.py index 0379e6f106d3ab6d9968b9770403ec9c8ef111ca..ad939a475d7dc8aa5ee1973c826b8ea53a6d55a8 100644 --- a/cve-py/gitwebtask/yamltask.py +++ b/cve-py/gitwebtask/yamltask.py @@ -55,7 +55,7 @@ class Task(object): """ if os.path.exists("sigs.yaml"): with open("sigs.yaml", "r") as f: - data = yaml.load(f) + data = yaml.safe_load(f) if data and data['sigs']: mysql = MysqlConn.Mysql() for sig in data['sigs']: diff --git a/cve-py/newexcels/20.03-CVE_2020-11-26.xlsx b/cve-py/newexcels/20.03-CVE_2020-11-26.xlsx deleted file mode 100644 index 1ed659b011ec0c6d8c825d6b6396ed695064c4e4..0000000000000000000000000000000000000000 Binary files a/cve-py/newexcels/20.03-CVE_2020-11-26.xlsx and /dev/null differ diff --git a/cve-py/newexcels/CVE-LTS-20.03-SP1-12-9.xlsx b/cve-py/newexcels/CVE-LTS-20.03-SP1-12-9.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..f91c4bbabdebee3f8bea54092b33c002a2343c61 Binary files /dev/null and b/cve-py/newexcels/CVE-LTS-20.03-SP1-12-9.xlsx differ diff --git a/cve-py/newexcels/cve-issue_2020-11-25.xlsx b/cve-py/newexcels/cve-issue_2020-11-25.xlsx deleted file mode 100644 index 3f58d6fb3d33db2c2976a9aa748aeef552b0a150..0000000000000000000000000000000000000000 Binary files a/cve-py/newexcels/cve-issue_2020-11-25.xlsx and /dev/null differ diff --git a/cve-py/newexcels/cve_issue_2020-11-26.xlsx b/cve-py/newexcels/cve_issue_2020-11-26.xlsx deleted file mode 100644 index f7c12d6140def0803a44d82e1b1fd66a79ccc980..0000000000000000000000000000000000000000 Binary files a/cve-py/newexcels/cve_issue_2020-11-26.xlsx and /dev/null differ diff --git a/cve-py/requirements.txt b/cve-py/requirements.txt index 80d582a398a963289f71bc22105c109807d65c1e..964d0c3f287063d9333bb98c6d4d3bab6411671e 100644 --- a/cve-py/requirements.txt +++ b/cve-py/requirements.txt @@ -1,10 +1,18 @@ APScheduler==3.6.3 DBUtils==1.3 +Flask==1.1.2 +Flask-SQLAlchemy==2.4.4 +Jinja2==2.11.2 +MarkupSafe==1.1.1 PyMySQL==0.10.1 PyYAML==5.3.1 +SQLAlchemy==1.3.20 +Werkzeug==1.0.1 certifi==2020.6.20 chardet==3.0.4 +click==7.1.2 idna==2.10 +itsdangerous==1.1.0 lxml==4.6.1 mysql==0.0.2 mysql-connector-python==8.0.22 @@ -13,7 +21,7 @@ pip==20.2.2 protobuf==3.13.0 pytz==2020.1 requests==2.24.0 -setuptools ==47.1.0 +setuptools==47.1.0 six==1.15.0 tzlocal==2.1 urllib3==1.25.11 diff --git a/cve-py/spec_error_excels/CVE.xlsx b/cve-py/spec_error_excels/CVE.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..eca072a3a17a35843dea8bc4b2b6fcdab6312130 Binary files /dev/null and b/cve-py/spec_error_excels/CVE.xlsx differ diff --git a/cve-py/tabletask/crawltask.py b/cve-py/tabletask/crawltask.py index e12189df69673e4669474877036bd9a5aaef452e..f7b8affadfcfc11a92aeb99a7f5e55375024400f 100644 --- a/cve-py/tabletask/crawltask.py +++ b/cve-py/tabletask/crawltask.py @@ -69,9 +69,9 @@ def crawling(url): repair_time = datetime.strptime(repair_time, '%m/%d/%Y') html1 = etree.HTML(element[0]) cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score-severity"]/text()') - [0].strip()) + [0].strip()).capitalize() nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv2-base-score"]/text()')[0].strip()) - vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv2-vector"]/text()')[0]).\ + vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv2-vector"]/text()')[0]). \ replace("(", "").replace(")", "").strip() access_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv2-av"]/text()')[0].strip()) access_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv2-ac"]/text()')[0].strip()) @@ -101,9 +101,10 @@ def crawling(url): else: element = html.xpath('//*[@id="cnaV3MetricHidden"]/@value') html1 = etree.HTML(element[0]) - cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score-severity"]/text()')[0].strip()) + cve_level = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score-severity"]/text()')[0] + .strip()).capitalize() nvd_score = str(html1.xpath('//*[@data-testid="vuln-cvssv3-base-score"]/text()')[0].strip()) - vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv3-vector"]/text()')[0]).replace("(", '').\ + vector_value = str(html1.xpath('//*[@data-testid="vuln-cvssv3-vector"]/text()')[0]).replace("(", ''). \ replace(')', '').strip() attack_vector = str(html1.xpath('//*[@data-testid="vuln-cvssv3-av"]/text()')[0].strip()) attack_complexity = str(html1.xpath('//*[@data-testid="vuln-cvssv3-ac"]/text()')[0].strip()) diff --git a/cve-py/tabletask/exceltask.py b/cve-py/tabletask/exceltask.py index 1354ad7618157808d246cc5afd4917ffd3b91fc5..b263ac879bf5284fedf19a644e70e75e8b567761 100644 --- a/cve-py/tabletask/exceltask.py +++ b/cve-py/tabletask/exceltask.py @@ -85,27 +85,6 @@ def crawl_cve_version(file_name): return cve_version -def crawl_scope_type(file_name): - """ - CVE type - :param file_name:excel file - :return scope_type:list - """ - scope_type = [] - try: - data = xlrd.open_workbook("./newexcels/" + file_name) - table_one = data.sheet_by_name("Sheet1") - row_number = table_one.nrows - for i in range(1, row_number): - scope_type_one = 'v' + str(table_one.cell(i, 4).value) - scope_type.append(scope_type_one) - except IndexError as e: - print("Subscript out of bounds", e) - except xlrd.XLRDError as e: - print("Form not found：Sheet1", e) - return scope_type - - def crawl_packname(file_name): """ Component diff --git a/cve-py/tabletask/mappingtask.py b/cve-py/tabletask/mappingtask.py index d1b4c01be6b0e9bb2915d1cf4d59054f63429c21..fa71403ad15cd38e2840c67f6051482e55df135e 100644 --- a/cve-py/tabletask/mappingtask.py +++ b/cve-py/tabletask/mappingtask.py @@ -90,7 +90,7 @@ def to_mysql(): if str(result['packname']) == pack_name_list[j]: if str(result["cpe_packname"]) != cpe_pack_name_list[j]: print("update data:" + pack_name_list[j], cpe_pack_name_list[j]) - sql = "update cve_package_cpe set cpe_packname=%s,createtime=%s where packname=%s" + sql = "update cve_package_cpe set cpe_packname=%s, create_time=%s where packname=%s" val = (cpe_pack_name_list[j], create_time, pack_name_list[j]) mysql.update(sql, val) mysql.dispose() diff --git a/cve-py/tabletask/runtask.py b/cve-py/tabletask/runtask.py index 184dd45eb973526ce51c55e012155bc6fff83548..ced6ddc10cb19711d38e73e78e17fb64d915b596 100644 --- a/cve-py/tabletask/runtask.py +++ b/cve-py/tabletask/runtask.py @@ -69,10 +69,10 @@ def handle_data(): # Crawler web data crawl_list = crawltask.crawling(urls[i]) # Determine whether the database content is the latest data - if result_dict["nvd_score"]: + if crawl_list[0]: if str(result_dict["nvd_score"]) == str(crawl_list[0]) and str(result_dict["vector_value"] ) == str(crawl_list[4]): - if result_dict['cve_status'] in [3, 4, 5]: + if result_dict['cve_status'] in [3, 4, 7]: print("update data:" + cve_num) update_time = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) try: @@ -147,6 +147,15 @@ def handle_data(): except IndexError as e: print("Subscript out of bounds", e) mysql.dispose(0) + sql = "select * from cve_spec_error where cve_num = %s" + val = (cve_num,) + result_spec_error = mysql.getOne(sql, val) + if result_spec_error: + print("过滤,修改status为6：{}".format(cve_num)) + sql = "update cve_origin_excel set cve_desc = %s, cve_status = %s where cve_num = %s" + val = (result_spec_error["cve_desc"], 6, cve_num) + mysql.update(sql, val) + mysql.dispose() exceltask.move_file(fileName) mysql.close() else: diff --git a/cve-py/tabletask/spec_error_task.py b/cve-py/tabletask/spec_error_task.py new file mode 100644 index 0000000000000000000000000000000000000000..378056622e7e64cf1533d4b7e46bdf9eb1ee8c76 --- /dev/null +++ b/cve-py/tabletask/spec_error_task.py @@ -0,0 +1,103 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Interaction between file and mysql data +Authors: xiaojianghui +Date: 12/08/2020 11:01 AM +""" + +import os +import xlrd +import time +import urllib.parse +from sqlalchemy import Column, String, create_engine, Integer, Boolean +from sqlalchemy.orm import sessionmaker +from sqlalchemy.ext.declarative import declarative_base +from downloadtask import downloadfiletask +from dbConnecttion import config + +Base = declarative_base() + + +class Details(Base): + __tablename__ = 'cve_spec_error' + id = Column(Integer, primary_key=True) + cve_num = Column(String) + cve_desc = Column(String) + cve_owner = Column(String) + cve_status = Column(Boolean) + pack_name = Column(String) + create_time = Column(String) + update_time = Column(String) + delete_time = Column(String) + + +def cur_date(): + """ + current date + Returns:createTime:string + + """ + create_time = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) + return create_time + + +def parse_error_excels(file_name): + """ + parse spec_error_excels files + Returns: + + """ + cve_num_list = [] + cve_desc_list = [] + cve_owner_list = [] + pack_name_list = [] + try: + data = xlrd.open_workbook('./spec_error_excels/' + file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + cve_num_list.append(table_one.cell(i, 1).value) + cve_desc_list.append(table_one.cell(i, 4).value) + cve_owner_list.append(table_one.cell(i, 2).value) + pack_name_list.append(table_one.cell(i, 3).value) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + return cve_num_list, cve_desc_list, cve_owner_list, pack_name_list + + +def add_error_details(): + """ + Conditions for filtering incorrect data + Returns: + + """ + downloadfiletask.download_spec_error_excels() + files = os.listdir('./spec_error_excels') + for file in files: + result = parse_error_excels(file) + print('开始批量插入人工CVE过滤条件') + objects = [] + pwd = urllib.parse.quote_plus(config.DBPWD) + engine = create_engine( + "mysql://" + config.DBUSER + ":" + pwd + "@" + config.DBHOST + "/" + config.DBNAME + "?charset=utf8") + for i in range(0, len(result[0])): + objects.append(Details(cve_num=result[0][i], cve_desc=result[1][i], cve_owner=result[2][i], + cve_status=1, pack_name=result[3][i], create_time=cur_date())) + db_session = sessionmaker(bind=engine) + session = db_session() + try: + session.bulk_save_objects(objects) + except Exception: + session.rollback() + session.commit() + session.close() + os.remove('./spec_error_excels/' + file) diff --git a/cve-py/tabletask/toexcel.py b/cve-py/tabletask/toexcel.py index 1e04c2069b59ec3a94cc280daca855fa310b1bdd..0126466d5287eba9fce6220bf2d29ee52b27dfa3 100644 --- a/cve-py/tabletask/toexcel.py +++ b/cve-py/tabletask/toexcel.py @@ -43,7 +43,7 @@ class MysqlToExcel(object): :return results: list """ mysql = Mysql() - sql = "select * from cve_origin_excel where cve_status = 3 or cve_status=4" + sql = "select * from cve_origin_excel where cve_status = 3 or cve_status=4 and is_import = 1" results = mysql.getMany(sql) print(results) mysql.dispose() @@ -126,10 +126,10 @@ class MysqlToExcel(object): @staticmethod def update_status(): """ - Change the data with status 3 and 4 to 5 to indicate that it has been exported in excel + Change the data with is_export 1 to 2 to indicate that it has been exported in excel """ mysql = Mysql() - sql = "update cve_origin_excel set cve_status=5 where cve_status=3 or cve_status=4" + sql = "update cve_origin_excel set is_export=2 where cve_status=3 or cve_status=4" mysql.update(sql) mysql.dispose() mysql.close() diff --git a/models/cve.go b/models/cve.go index 037cc89b49ab4a0827b804ec5a90f7219dd8274d..ca362bc19ee1e680cb36a5bd00f5491c4448dfe4 100644 --- a/models/cve.go +++ b/models/cve.go @@ -32,6 +32,40 @@ func UpdateCveIssueStatusById(status int, issueId int64) (id int64) { return } +func LockUpdateCveIssueStatus(status int, issueId int64) (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_gite_origin_issue SET proc_status = ?, issue_update_at = ? where id = ? and proc_status = 0", + status, common.GetCurTime(), issueId).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("LockUpdateCveIssueStatus, update cve_gite_origin_issue row affected nums: ", num, ",issueId: ", issueId) + return true + } + return false + } else { + logs.Error("LockUpdateCveIssueStatus, update failed, cve_gite_origin_issue, ", ",issueId: ", issueId, ", err: ", err) + return false + } +} + +func UnLockUpdateCveIssueStatus(beforeStatus, afterStatus int8, updateTime string) (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_gite_origin_issue SET proc_status = ?, issue_update_at = ? where proc_status = ?", + afterStatus, updateTime, beforeStatus).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UnLockUpdateCveIssueStatus, update cve_gite_origin_issue row affected nums: ", num) + return true + } + return false + } else { + logs.Error("UnLockUpdateCveIssueStatus, update failed, cve_gite_origin_issue, err: ", err) + return false + } +} + func QueryOriginCveIssue(prcnum int) ([]GiteOriginIssue, int64, error) { o := orm.NewOrm() var gs []GiteOriginIssue @@ -590,6 +624,45 @@ func UpdateOriginStatus(updateTime, pakName, version string, cveId int64, status } } +func LockUpdateOriginStatus(updateTime, pakName, version string, cveId int64, status int) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_upstream SET "+ + "cve_status = ?, update_time = ? where cve_id = ? and git_packname = ? and version = ? and cve_status in (0,1)", + status, updateTime, cveId, pakName, version).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("cve_origin_upstream row affected nums: ", + num, ",cveId: ", cveId, ",", updateTime, pakName, version) + return true + } + return false + } else { + logs.Error("更新失败, cve_origin_upstream, ", ",cveId: ", + cveId, ",", updateTime, pakName, version, ", err: ", err) + return false + } +} + +func UnLockUpdateOriginStatus(beforeStatus, afterStatus int, updateTime string) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_upstream SET "+ + "cve_status = ?, update_time = ? where cve_status = ?", + afterStatus, updateTime, beforeStatus).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UnLockUpdateOriginStatus, cve_origin_upstream row affected nums: ", + num) + return true + } + return false + } else { + logs.Error("UnLockUpdateOriginStatus, update failed, cve_origin_upstream, err: ", err) + return false + } +} + func UpdateOriginExcelStatus(updateTime string, cveId int64, status int) bool { o := orm.NewOrm() res, err := o.Raw("UPDATE cve_origin_excel SET "+ @@ -607,6 +680,42 @@ func UpdateOriginExcelStatus(updateTime string, cveId int64, status int) bool { } } +func LockUpdateOriginExcelStatus(updateTime string, cveId int64, status int) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_excel SET "+ + "cve_status = ?, update_time = ? where cve_id = ? and cve_status in (0,1)", status, updateTime, cveId).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("cve_origin_excel row affected nums: ", num, ",cveId: ", cveId, ",", updateTime) + return true + } + return false + } else { + logs.Error("更新失败, cve_origin_excel, ", ",cveId: ", cveId, ",", updateTime, ", err: ", err) + return false + } +} + +func UnLockUpdateOriginExcelStatus(beforeStatus, afterStatus int, updateTime string) bool { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_origin_excel SET "+ + "cve_status = ?, update_time = ? where cve_status = ?", + afterStatus, updateTime, beforeStatus).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UnLockUpdateOriginExcelStatus, cve_origin_excel row affected nums: ", + num) + return true + } + return false + } else { + logs.Error("UnLockUpdateOriginExcelStatus, update failed, cve_origin_excel, err: ", err) + return false + } +} + func UpdateVulnCenter(cve *VulnCenter, field ...string) bool { o := orm.NewOrm() _, err := o.Update(cve, field...) @@ -738,7 +847,8 @@ WHERE a.issue_num = ? func GetCanExportCveDataSameNum(cId string) (list []ExcelExport, err error) { sql := `SELECT a.cve_id,a.owned_component,a.cve_brief,c.*,d.sec_id, d.introduction,d.summary,d.theme,d.description,d.influence_component, -d.affect_product,d.reference_link,d.affect_status,e.public_date,e.openeuler_sa_num +d.affect_product,d.reference_link,d.affect_status, +e.public_date,e.openeuler_sa_num,a.cve_level FROM cve_issue_template a LEFT JOIN cve_score c ON c.cve_id = a.cve_id diff --git a/models/excel.go b/models/excel.go index 20dc9a0a75e0fe5ff766dc6df5e31938e24002ef..807f4e9a65496f6cb0442f7ad2aca4560675a794 100644 --- a/models/excel.go +++ b/models/excel.go @@ -1,6 +1,9 @@ package models -import "github.com/astaxie/beego/orm" +import ( + "github.com/astaxie/beego/logs" + "github.com/astaxie/beego/orm" +) //ExcelExport the export excel row content model type ExcelExport struct { @@ -55,3 +58,16 @@ func (er *ExportRecord) Read(field ...string) error { o := orm.NewOrm() return o.Read(er, field...) } + +func (elt *EmailList) Read(field ...string) ([]EmailList, error) { + o := orm.NewOrm() + var el []EmailList + var num int64 + num, err := o.Raw("select *"+ + " from cve_email_list where email_type = ?", elt.EmailType).QueryRows(&el) + if err == nil && num > 0 { + return el, nil + } + logs.Error("查询 cve_email_list ,err: ", err) + return el, err +} diff --git a/models/issue.go b/models/issue.go index a08b8409835f573a261a76f99a416565da476eb7..4f925f8a35c18071ea352d49fb08c17a4fcefea6 100644 --- a/models/issue.go +++ b/models/issue.go @@ -155,20 +155,6 @@ func QueryIssueScoreRecord(cveId int64, status int8) (ScoreRecord, error) { return sr, err } -func GetIssueTemplet(it *IssueTemplate) (localIt IssueTemplate, value bool) { - o := orm.NewOrm() - cveId := it.CveId - err := o.Raw("select *"+ - " from cve_issue_template where cve_id = ? ", cveId).QueryRow(it) - if err == nil { - logs.Info("cve_issue_template 查询结果：", it) - return *it, true - } else { - logs.Info("查询 cve_issue_template err, cveId: ", it.CveId, "err: ", err) - return *it, false - } -} - func GetIssueTemplateByColName(it *IssueTemplate, colName ...string) error { o := orm.NewOrm() err := o.Read(it, colName...) @@ -224,7 +210,6 @@ func UpdatePackageByCveId(pkgStr string, cveId int64) error { } } return nil - } return nil } @@ -329,6 +314,20 @@ func CreateIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { } } +func GetAllIssueHook(hookId int64, count, status int) (localh []IssueHooks, value bool) { + o := orm.NewOrm() + var localIh []IssueHooks + var num int64 + num, err := o.Raw("select *"+ + " from cve_issue_hooks where status = ? and id > ? order by id limit ?", status, hookId, count).QueryRows(&localIh) + if err == nil && num > 0 { + return localIh, true + } else { + logs.Info("查询 cve_issue_hooks err, err: ", err) + return localIh, false + } +} + func GetIssueHook(ih *IssueHooks) (localh []IssueHooks, value bool) { o := orm.NewOrm() var localIh []IssueHooks @@ -411,12 +410,52 @@ func UpdateSecNotice(sec *SecurityNotice) (secId int64, err error) { } } -func UpdateIssueStatus(iss VulnCenter, status int8) (secId int64) { +func UpdateIssueStatus(iss VulnCenter, status int8) { o := orm.NewOrm() _ = o.Raw("UPDATE cve_vuln_center SET cve_status = ? WHERE cve_id = ? and cve_num = ?", status, iss.CveId, iss.CveNum).QueryRow() return } +func GetIssueSpecErrByColName(se *SpecError, colName ...string) error { + o := orm.NewOrm() + err := o.Read(se, colName...) + return err +} + +func LockUpdateIssueStatus(cveId int64, cveNum string, status int8) (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_vuln_center SET cve_status = ? WHERE cve_id = ? and cve_num = ? and cve_status in (0,1)", + status, cveId, cveNum).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("update cve_vuln_center row affected nums: ", num, ",cveId: ", cveId) + return true + } + return false + } else { + logs.Error("update failed, cve_vuln_center, ", ",cveId: ", cveId, ", err: ", err) + return false + } +} + +func UnLockUpdateIssueStatus(beforeStatus, afterStatus int8, updateTime string) (bool) { + o := orm.NewOrm() + res, err := o.Raw("UPDATE cve_vuln_center SET cve_status = ?,update_time = ? WHERE cve_status = ?", + afterStatus, updateTime, beforeStatus).Exec() + if err == nil { + num, _ := res.RowsAffected() + if num > 0 { + logs.Info("UnLockUpdateIssueStatus, update cve_vuln_center row affected nums: ", num) + return true + } + return false + } else { + logs.Error("UnLockUpdateIssueStatus, update failed, cve_vuln_center, err: ", err) + return false + } +} + func UpdateIssueScore(iss VulnCenter, status int8) (id int64) { o := orm.NewOrm() _ = o.Raw("UPDATE cve_score SET n_score_status = ? WHERE cve_id = ? and cve_num = ?", status, iss.CveId, iss.CveNum).QueryRow() diff --git a/models/modeldb.go b/models/modeldb.go index cfb5c7f977b0af8e8d830e37bfb5ec62f0bde958..ee52ef52db0974ca62ff7497ad17968be8a7e32b 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -38,7 +38,7 @@ type VulnCenter struct { CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` - Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求)"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求);5:issue已经创建过，不符合要求，不处理"` CveVersion string `orm:"size(128);column(cve_version)" description:"cve归属版本"` RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` @@ -508,7 +508,8 @@ type OriginExcel struct { Integrity string `orm:"size(64);column(integrity);null"` Availability string `orm:"size(64);column(availability);null"` Authentication string `orm:"size(64);column(authentication);null"` - CveStatus int8 `orm:"default(0);column(cve_status);null" description:"0:未处理;1:已修改；2：已处理; 3: 数据错误，不做处理"` + CveStatus int8 `orm:"default(0);column(cve_status);null" description:"0:未处理;1:已修改；2：已处理; 3: 数据错误，不做处理; 4: 版本信息错误; 6: cve已经生成过issue; 7:cve年限不对"` + IsExport int8 `orm:"default(1);column(is_export);null" description:"1: 未导出; 2: 已导出"` CreateTime string `orm:"type(datetime);column(create_time);null"` UpdateTime string `orm:"type(datetime);column(update_time);null"` DeleteTime string `orm:"type(datetime);column(delete_time);null"` @@ -625,6 +626,18 @@ type SpecIssueAssigness struct { DeleteTime string `orm:"size(32);column(delete_time);null"` } +type SpecError struct { + Id int64 `orm:"pk;auto;column(id)"` + CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` + Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` + Owner string `orm:"size(256);column(cve_owner);index" description:"仓库所在组织"` + Status int8 `orm:"default(1);column(cve_status)" description:"1:正常数据;2:已删除"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + func CreateDb() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -647,7 +660,7 @@ func CreateDb() bool { new(OriginUpstreamFixSuggest), new(OriginUpstreamFixSuggestRefTag), new(OriginUpstreamFixSuggestRef), new(GiteOriginIssue), new(OriginExcel), new(ExportRecord), new(GitRepoGroups), new(GiteRepo), new(GiteRepoMember), new(GiteRepoBranch), new(PackageCpe), new(EmailList), new(IssueAssignee), - new(IssueRepoWhitelist), new(IssueCreateRecord), new(OpenGussYaml), new(SpecIssueAssigness), + new(IssueRepoWhitelist), new(IssueCreateRecord), new(OpenGussYaml), new(SpecIssueAssigness), new(SpecError), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/models/uploadcve.go b/models/uploadcve.go index d69f3aab7f2e74ae586cc2613b18d2b3fadf030d..042e9aeda2d6ff74d98ad138c300516e8f20d3f3 100644 --- a/models/uploadcve.go +++ b/models/uploadcve.go @@ -66,96 +66,67 @@ func QueryAssignee() ([]IssueAssignee, bool) { } func QueryRepoAllMaintainer(pkgName string) ([]GiteRepoMember, bool) { + sia := SpecIssueAssigness{PackageName: pkgName, Status: 1} gr, okrg := QueryOriginRepo(pkgName) if !okrg || gr.RepoId == 0 { logs.Error("maintainer query failed, pkgName: ", pkgName, ",okrg: ", okrg) + specError := GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + var grm []GiteRepoMember + var grmr GiteRepoMember + grmr.MemberName = sia.Assignee + grm = append(grm, grmr) + return grm, true + } return nil, false } grm, okgrm := QueryRepoMember(gr.GroupId, "Maintainer") if !okgrm { logs.Error("maintainer query failed, pkgName: ", pkgName, ",gr.GroupId: ", gr.GroupId, ",okgrm: ", okgrm) + specError := GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + var grmr GiteRepoMember + grmr.MemberName = sia.Assignee + grm = append(grm, grmr) + return grm, true + } return nil, false } - return grm, true -} - -func QueryRepoMaintainer(pkgName string, version string) (GitOpenEuler, bool) { - goe, ok := QueryYamlData(pkgName, version) - if !ok || goe.GitId == 0 { - gr, okrg := QueryOriginRepo(pkgName) - if !okrg || gr.RepoId == 0 { - aie, aieOk := QueryAssignee() - if aieOk { - for _, a := range aie { - if a.Assignee != "" && len(a.Assignee) > 1 { - goe.MainTainer = a.Assignee - break - } - } - return goe, true - } - return goe, ok - } - grm, okgrm := QueryRepoMember(gr.GroupId, "Maintainer") - if !okgrm { - aie, aieOk := QueryAssignee() - if aieOk { - for _, a := range aie { - if a.Assignee != "" && len(a.Assignee) > 1 { - goe.MainTainer = a.Assignee - break - } - } - return goe, true - } - return goe, ok - } - for _, g := range grm { - if g.MemberName != "" && len(g.MemberName) > 1 { - goe.MainTainer = g.MemberName + specError := GetSpecIssueAssignee(&sia, "package_name", "status") + if specError == nil && sia.Id > 0 { + assFlag := false + for _, mt := range grm { + if mt.MemberName == sia.Assignee { + assFlag = true break } } - return goe, true - } else { - if goe.MainTainer == "" || len(goe.MainTainer) == 0 { - gr, okrg := QueryOriginRepo(pkgName) - if !okrg || gr.RepoId == 0 { - aie, aieOk := QueryAssignee() - if aieOk { - for _, a := range aie { - if a.Assignee != "" && len(a.Assignee) > 1 { - goe.MainTainer = a.Assignee - break - } - } - return goe, true - } - return goe, ok - } - grm, okgrm := QueryRepoMember(gr.GroupId, "Maintainer") - if !okgrm { - aie, aieOk := QueryAssignee() - if aieOk { - for _, a := range aie { - if a.Assignee != "" && len(a.Assignee) > 1 { - goe.MainTainer = a.Assignee - break - } - } - return goe, true - } - return goe, ok - } - for _, g := range grm { - if g.MemberName != "" && len(g.MemberName) > 1 { - goe.MainTainer = g.MemberName - break - } - } + if !assFlag { + var grmr GiteRepoMember + grmr.MemberName = sia.Assignee + grm = append(grm, grmr) } } - return goe, true + return grm, true +} + +func QueryRepoMaintainer(pkgName string) (string, bool) { + gr, okrg := QueryOriginRepo(pkgName) + if !okrg || gr.RepoId == 0 { + logs.Error("repo does not exist, err: ", okrg) + return "", okrg + } + grm, okgrm := QueryRepoMember(gr.GroupId, "Maintainer") + if !okgrm { + logs.Error("Maintainer does not exist, err: ", okrg) + return "", okgrm + } + for _, g := range grm { + if g.MemberName != "" && len(g.MemberName) > 1 { + return g.MemberName, true + } + } + return "", false } func QueryCveOpeneulerdata(pkgName string, version string) (GitOpenEuler, bool) { diff --git a/task/cve.go b/task/cve.go index e1be20e167dcccba0009ab8aac55027599f68b5a..b5046ccace4f2ac3067a2e91400a4475ba3a9918 100644 --- a/task/cve.go +++ b/task/cve.go @@ -31,6 +31,7 @@ func ProcCveOriginData(prcNum, days, openeulerNum int, cveRef, owner string) (bo //ParamsCveOriginData Processing cve raw data func ParamsCveOriginData() error { defer common.Catchs() + logs.Info("将cve原始数据生成cve库 task start") // Query the cve to be processed, 1: add; 2: modify BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -59,5 +60,6 @@ func ParamsCveOriginData() error { owner := BConfig.String("gitee::owner") // Get the data source of the table _, errx := ProcCveOriginData(prcnum, days, openeulernum, cveRef, owner) + logs.Info("将cve原始数据生成cve库 task end") return errx } diff --git a/task/genexcel.go b/task/genexcel.go index db53e82631d4d6e03a3733b9b79449ce39f652ff..fdca3ff96b5e1e932e49c6af4006f6dd04cad185 100644 --- a/task/genexcel.go +++ b/task/genexcel.go @@ -11,6 +11,7 @@ import ( //GenExcelData generate excel data func GenExcelData() error{ defer common.Catchs() + logs.Info("执行issue生成excel task start") // 查询需要处理的cve, 1:新增；2：修改 BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil{ @@ -35,5 +36,6 @@ func GenExcelData() error{ //excelName := "openEuler." + common.GetCurDate() + ".xlsx" //errx := taskhandler.GenerateCveExcel(excelName, snPrefix, snSuffix, forceRewrite) errx := taskhandler.GenerateExcelTask() + logs.Info("执行issue生成excel task end") return errx } \ No newline at end of file diff --git a/task/grabissue.go b/task/grabissue.go index a7ea7d536cc4152f2ccbd7e1dc7306a0ad8a3539..6980137984fd591a41b26d0891936b283370b564 100644 --- a/task/grabissue.go +++ b/task/grabissue.go @@ -12,6 +12,7 @@ import ( //GetIssueData get the issue data func GetIssueData() error{ defer common.Catchs() + logs.Info("获取已经创建的issue数据源 task start") // 查询需要处理的cve, 1:新增；2：修改 BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil{ @@ -29,5 +30,6 @@ func GetIssueData() error{ return errors.New("获取token失败, 无法获取issue任务") } errx := taskhandler.GrabIssueByOrg(giteeToken, owner) + logs.Info("获取已经创建的issue数据源 task end") return errx } \ No newline at end of file diff --git a/task/hook.go b/task/hook.go new file mode 100644 index 0000000000000000000000000000000000000000..9f009cef6e86c1388a57dc21cb3e04b7d2857b92 --- /dev/null +++ b/task/hook.go @@ -0,0 +1,28 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "github.com/astaxie/beego/config" + "github.com/astaxie/beego/logs" + "os" +) + +func ProcHookEvent() error { + defer common.Catchs() + logs.Info("处理hook事件 task start") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return err + } + hookurl := BConfig.String("hook::hookurl") + owner := BConfig.String("gitee::owner") + accessToken := os.Getenv("GITEE_TOKEN") + pwd := BConfig.String("hook::hookpwd") + // Get the data source of the table + err = taskhandler.ProcHookEvent(hookurl, owner, accessToken, pwd) + logs.Info("处理hook事件 task end") + return err +} diff --git a/task/inittask.go b/task/inittask.go index 4b6667b6ef3840eae3b6d85aa87a800e4f88c0e5..b292e22d8eb3156f5d1e7e3df6a91725286fd15c 100644 --- a/task/inittask.go +++ b/task/inittask.go @@ -54,6 +54,22 @@ func CreatTask(createIssue string) { logs.Info("执行创建issue task end") } +// unlock table +func UnlockTask(unLockTable string) { + logs.Info("执行解锁表 task start") + cUnLockTable := toolbox.NewTask("UnLockTable", unLockTable, UnLockTable) + toolbox.AddTask("UnLockTable", cUnLockTable) + logs.Info("执行解锁表 task end") +} + +// Handling hook events +func ProcHookTask(prochook string) { + logs.Info("执行处理Hook事件 task start") + cProchook := toolbox.NewTask("ProcHookEvent", prochook, ProcHookEvent) + toolbox.AddTask("ProcHookEvent", cProchook) + logs.Info("执行处理Hook事件 task end") +} + //GenSAExcelTask Execute issue to generate excel task start func GenSAExcelTask(genExcel string) { logs.Info("执行issue生成excel task start") @@ -132,7 +148,7 @@ func InitTask() bool { createIssue := BConfig.String("crontab::createissue") CreatTask(createIssue) } else { - logs.Info("no task") + logs.Info("createissue no task") } // Generate excel data from issue genexcelflag, errx := BConfig.Int("crontab::genexcelflag") @@ -140,7 +156,23 @@ func InitTask() bool { genExcel := BConfig.String("crontab::genexcel") GenSAExcelTask(genExcel) } else { - logs.Info("no task") + logs.Info("genexcel no task") + } + // unlock mysql + unlockflag, errx := BConfig.Int("crontab::unlockflag") + if unlockflag == 1 && errx == nil { + unlock := BConfig.String("crontab::unlock") + UnlockTask(unlock) + } else { + logs.Info("unlock no task") + } + // Handling hook events + hookflag, errx := BConfig.Int("crontab::hookflag") + if hookflag == 1 && errx == nil { + prochook := BConfig.String("crontab::prochook") + ProcHookTask(prochook) + } else { + logs.Info("prochook no task") } return true } diff --git a/task/issuetask.go b/task/issuetask.go index b65294c19f3198c60f1e07718dce247fd357e8cd..3ea432422a10ac1ad0dd82cf6dee083da6606da7 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -78,30 +78,43 @@ func GenIssueRecordData(icr *models.IssueCreateRecord, issueValue models.VulnCen var mutex sync.Mutex func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears int) error { + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("issue token Get failed, current time: ", common.GetCurTime()) + return errors.New("Failed to get token in environment variable") + } + owner := beego.AppConfig.String("gitee::owner") + path := beego.AppConfig.String("gitee::path") for ; ; { cveData, err := models.QueryIssue(beforeTime, prcnum) if err == nil && len(cveData) > 0 { logs.Info(cveData) } else { - logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) - return err - } - accessToken := os.Getenv("GITEE_TOKEN") - if accessToken == "" || len(accessToken) < 1 { - logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) + logs.Info("No cve data can be used, current time: ", common.GetCurTime(), ", err: ", err) return err } - owner := beego.AppConfig.String("gitee::owner") - path := beego.AppConfig.String("gitee::path") for index, issueValue := range cveData { - logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) + logs.Info("Currently processing：", index, ",cve data, cveNum: ", issueValue.CveNum) + // add mutex + lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) + if !lockErr { + logs.Error("The current cve is processing, continue to process the next data, err: ", lockErr, ",data: ", issueValue) + continue + } + se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} + seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") + if seError == nil && se.Id > 0 { + models.UpdateIssueStatus(issueValue, 5) + logs.Info("The current issue does not need to be processed, it has been processed, cveData: ", issueValue) + continue + } // Determine whether the issue has been processed goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) if oks { if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || goi.State == "已完成" || goi.State == "已拒绝" { models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) + logs.Info("The cve data has already been submitted to the issue, no need to submit repeatedly, cveData: ", issueValue) continue } } @@ -118,11 +131,11 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears if cveYears <= years { icr := models.IssueCreateRecord{} models.UpdateIssueStatus(issueValue, 4) - logs.Info("cve: ", issueValue.CveNum, ",需要大于： ", - years, ",否则不需要在git上提交issue, cveData: ", issueValue) + logs.Info("cve: ", issueValue.CveNum, ",Need to be greater than： ", + years, ",Otherwise, there is no need to submit an issue on git, cveData: ", issueValue) sc, err := models.QueryIssueScore(issueValue.CveId) if err != nil || sc.Id == 0 { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + logs.Error("Failed to get Score, err: ", err, "cveId: ", issueValue.CveId) continue } GenIssueRecordData(&icr, issueValue, sc, 2) @@ -140,24 +153,27 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) if issueExist { models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) + logs.Info("The cve data has been displayed on the official website, " + + "no need to submit an issue on git, cveData: ", issueValue) continue } // Process each piece of cve data if issueValue.Status == 0 { + issueValue.Status = 2 mutex.Lock() err := ProcIssue(issueValue, accessToken, owner, path) mutex.Unlock() if err != nil { - logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + logs.Error("Failed to create issue, cvenum: ", issueValue.CveNum, "err,err: ", err) continue } } else { + issueValue.Status = 2 mutex.Lock() err := ProcUpdateIssue(issueValue, accessToken, owner, path) mutex.Unlock() if err != nil { - logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + logs.Error("Failed to update issue, cvenum: ", issueValue.CveNum, "err,err: ", err) continue } } @@ -165,7 +181,7 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears icr := models.IssueCreateRecord{} sc, err := models.QueryIssueScore(issueValue.CveId) if err != nil || sc.Id == 0 { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + logs.Error("Failed to get Score, err: ", err, "cveId: ", issueValue.CveId) continue } GenIssueRecordData(&icr, issueValue, sc, 1) @@ -180,6 +196,13 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears } func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears int) error { + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) + return errors.New("Failed to get token in environment variable") + } + owner := beego.AppConfig.String("gitee::owner") + path := beego.AppConfig.String("gitee::path") issueWhiteData, issueErr := models.QueryIssueWhitelist(1) if issueErr == nil && len(issueWhiteData) > 0 { for _, issues := range issueWhiteData { @@ -191,15 +214,21 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) break } - accessToken := os.Getenv("GITEE_TOKEN") - if accessToken == "" || len(accessToken) < 1 { - logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime(), ",err: ", err) - break - } - owner := beego.AppConfig.String("gitee::owner") - path := beego.AppConfig.String("gitee::path") for index, issueValue := range cveData { logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) + // add mutex + lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) + if !lockErr { + logs.Error("The current cve is processing, continue to process the next data, err: ", lockErr, ",data: ", issueValue) + continue + } + se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} + seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") + if seError == nil && se.Id > 0 { + models.UpdateIssueStatus(issueValue, 5) + logs.Info("The current issue does not need to be processed, it has been processed, cveData: ", issueValue) + continue + } // Determine whether the issue has been processed goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) if oks { @@ -293,6 +322,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i //CreateIssue Create issue func CreateIssue() error { defer common.Catchs() + logs.Info("执行创建issue task start") // Query the cve to be processed, 1: add; 2: modify BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -333,10 +363,12 @@ func CreateIssue() error { if issueWhitelist == 1 { limitErr := addLimitedIssue(beforeTime, prcnum, years, toolYears, manYears) logs.Info("addLimitedIssue, err: ", limitErr) + logs.Info("执行创建issue task end") return limitErr } else { unlimitErr := addUnlimitedIssue(beforeTime, prcnum, years, toolYears, manYears) logs.Info("addUnlimitedIssue, err: ", unlimitErr) + logs.Info("执行创建issue task end") return unlimitErr } } @@ -352,15 +384,15 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri // Query issue template var it models.IssueTemplate it.CveId = issueValue.CveId - _, bools := models.GetIssueTemplet(&it) - if bools && it.TemplateId > 0 { + templateErr := models.GetIssueTemplateByColName(&it, "cve_id") + if templateErr == nil && it.TemplateId > 0 { it.NVDScore = sr.NVDScore it.NVDVector = sr.NvectorVule it.CveBrief = issueValue.Description it.CveLevel = issueValue.CveLevel if it.Assignee == "" || len(it.Assignee) < 2 { - gitYaml, ok := models.QueryRepoMaintainer(issueValue.PackName, issueValue.CveVersion) - if !ok || gitYaml.MainTainer == "" || len(gitYaml.MainTainer) < 1 { + repoMainTainer, ok := models.QueryRepoMaintainer(issueValue.PackName) + if !ok || repoMainTainer == "" || len(repoMainTainer) < 1 { assignee, assErr := taskhandler.GetCollaboratorInfo(accessToken, owner, path) if assignee != "" && len(assignee) > 1 { it.Assignee = assignee @@ -370,7 +402,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri ", cveid: ", issueValue.CveId, ",创建无maintainer的issue") } } else { - it.Assignee = gitYaml.MainTainer + it.Assignee = repoMainTainer } } path = it.Repo @@ -406,8 +438,8 @@ func ProcIssue(issueValue models.VulnCenter, accessToken, owner, path string) er path = issueValue.PackName logs.Info("查询安全信息：sn: ", sn) // Get issue handler - gitYaml, ok := models.QueryRepoMaintainer(issueValue.PackName, issueValue.CveVersion) - if !ok || gitYaml.MainTainer == "" || len(gitYaml.MainTainer) < 1 { + repoMainTainer, ok := models.QueryRepoMaintainer(issueValue.PackName) + if !ok || repoMainTainer == "" || len(repoMainTainer) < 1 { assignee, err = taskhandler.GetCollaboratorInfo(accessToken, owner, path) if assignee == "" { logs.Error("获取仓库： owner:", owner, "path:", path, "分析人失败", "err:", err, @@ -415,16 +447,7 @@ func ProcIssue(issueValue models.VulnCenter, accessToken, owner, path string) er //return err } } else { - assignee = gitYaml.MainTainer - // Query the affected branch information - gt, errgt := models.QueryCveBrands(gitYaml.GitId) - if errgt == nil { - for _, t := range gt { - if t.Brands != "" && t.Brands != "master" && len(t.Brands) > 1 { - brandArry = append(brandArry, t.Brands) - } - } - } + assignee = repoMainTainer } } else { logs.Error("获取security 失败, err: ", err, "cveId: ", issueValue.CveId) @@ -447,7 +470,6 @@ func ProcIssue(issueValue models.VulnCenter, accessToken, owner, path string) er for _, branch := range branchList { brandArry = append(brandArry, branch) } - } resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, brandArry) if err != nil { diff --git a/task/oricvecheck.go b/task/oricvecheck.go index 33ceab31f468c45f78507db765124c4811c79867..7e57b60706c23647f082fff5cefe792c30d3873a 100644 --- a/task/oricvecheck.go +++ b/task/oricvecheck.go @@ -2,6 +2,7 @@ package task import ( "cvevulner/common" + "cvevulner/models" "cvevulner/taskhandler" "errors" "github.com/astaxie/beego/config" @@ -11,6 +12,7 @@ import ( //CheckOriCve Check the CVE of the original data func CheckOriCve() error { defer common.Catchs() + logs.Info("校验原始cve数据 task start") // Query the cve to be processed, 1: add; 2: modify BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -25,5 +27,22 @@ func CheckOriCve() error { } // Get the data source of the table _, err = taskhandler.CheckCveOriginData(prcNum) + logs.Info("校验原始cve数据 task end") return err } + +// Unlock database table data +func UnLockTable() error { + defer common.Catchs() + logs.Info("执行解锁表 task start") + // unlock center + models.UnLockUpdateIssueStatus(15, 0, common.GetCurTime()) + // unlock origin cve + models.UnLockUpdateCveIssueStatus(15, 0, common.GetCurTime()) + // unlock upstream + models.UnLockUpdateOriginStatus(15, 0, common.GetCurTime()) + // unlock excel + models.UnLockUpdateOriginExcelStatus(15, 0, common.GetCurTime()) + logs.Info("执行解锁表 task end") + return nil +} diff --git a/task/printlog.go b/task/printlog.go index 5641ff6e7cc417f99b94acdeb54b196f8b098ffb..5f1afe6154069073d8423ce6739e7b8c55765173 100644 --- a/task/printlog.go +++ b/task/printlog.go @@ -39,10 +39,12 @@ func DeletLogs() error { // ProcLogData Processing log func ProcLogData() error { + logs.Info("创建日志任务 task start") // delete logs err := DeletLogs() if err != nil { logs.Error(err) } + logs.Info("创建日志任务 task end") return err } diff --git a/task/yaml.go b/task/yaml.go index 14580fc7fc89e0bba2e558faca2f87b6d4cac264..cd37b16126a901f88dabf1fb00197722ed12790e 100644 --- a/task/yaml.go +++ b/task/yaml.go @@ -11,6 +11,7 @@ import ( // GetYamlData Get yaml data source func GetYamlData() error { defer common.Catchs() + logs.Info("获取yaml数据源 task start") // Query the cve to be processed, 1: add; 2: modify BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -28,5 +29,6 @@ func GetYamlData() error { if err == nil { _, err = taskhandler.GetYamlByGit(apiUrl) } + logs.Info("获取yaml数据源 task end") return err } diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index 514fa709ea0f21168f372b6791f2de56a6c0f41d..57818d44dc8f8204df7b34cad9b5828a66eda6d0 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -94,14 +94,14 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st defer common.Catchs() var it models.IssueTemplate it.CveId = cve.CveId - _, err := models.GetIssueTemplet(&it) + templateErr := models.GetIssueTemplateByColName(&it, "cve_id") sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") if specError == nil && sia.Id > 0 { it.Assignee = sia.Assignee assignee = sia.Assignee } - if err && it.IssueNum != "" && len(it.IssueNum) > 0 { + if templateErr == nil && it.IssueNum != "" && len(it.IssueNum) > 0 { if it.Assignee == "" || len(it.Assignee) == 0 { it.Assignee = assignee } @@ -125,14 +125,14 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st resp, err := util.HTTPPatch(url, requestBody) updateLock.Unlock() if err != nil { - logs.Error("创建issue失败, cveNum: ", cve.CveNum, "err: ", err) + logs.Error("Update issue failed, cveNum: ", cve.CveNum, "err: ", err, ",url: ", url) return "", err } if _, ok := resp["id"]; !ok { - logs.Error("创建issue 失败, err: ", ok, "url: ", url) - return "", errors.New("创建issue失败") + logs.Error("Update issue failed, err: ", ok, "url: ", url) + return "", errors.New("Update issue failed") } - logs.Info("issue 创建成功，cveNum: ", cve.CveNum, "issueNum: ", resp["number"].(string)) + logs.Info("Update issue successfully, cveNum: ", cve.CveNum, "issueNum: ", resp["number"].(string)) // Structure data //var issueTemp models.IssueTemplate CreateIssueData(&it, cve, sc, resp, path, it.Assignee, issueType, labels, owner) @@ -147,12 +147,12 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st // Store issue data issTempID, err := models.CreateIssueTemplate(&it) if err != nil { - logs.Error("创建issue 模板的数据失败, cveNum: ", cve.CveNum, "err: ", err) + logs.Error("Failed to update data of issue template, cveNum: ", cve.CveNum, ",err: ", err) return "", err } - logs.Info("创建issue 模板的数据成功, issTempID: ", issTempID, "cveNum: ", cve.CveNum) + logs.Info("Successfully updated the data of the issue template, issTempID: ", issTempID, ",cveNum: ", cve.CveNum) } else { - logs.Info("不需要更新issue模板及issue状态, its: ", it) + logs.Info("No need to update the issue template and issue status, its: ", it) } // Update issue status models.UpdateIssueStatus(cve, 2) @@ -180,14 +180,14 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st resp, err := util.HTTPPost(url, requestBody) updateLock.Unlock() if err != nil { - logs.Error("url: ", url, "创建issue失败, cveNum: ", cve.CveNum, "err: ", err) + logs.Error("url: ", url, "Failed to create issue, cveNum: ", cve.CveNum, ", err: ", err, ",url:", url) models.DeleteIssueTemplate(issTempID) return "", err } if _, ok := resp["id"]; !ok { - logs.Error("创建issue 失败, err: ", ok, "url: ", url) + logs.Error("Failed to create issue, err: ", ok, "url: ", url) models.DeleteIssueTemplate(issTempID) - return "", errors.New("创建issue失败") + return "", errors.New("Failed to create issue") } //var issueTemps models.IssueTemplate issueTemp.TemplateId = issTempID @@ -206,11 +206,11 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st // Store issue data issTempIDx, idxErr := models.UpdateIssueTemplateAll(&issueTemp) if idxErr != nil { - logs.Error("创建issue 模板的数据失败, cveNum: ", cve, ",err: ", err) + logs.Error("Failed to create data for issue template, cveNum: ", cve, ",err: ", err) //models.DeleteIssueTemplate(issTempID) return "", err } - logs.Info("创建issue 模板的数据成功, issTempID: ", issTempIDx, "cveNum: ", cve.CveNum) + logs.Info("The data of the issue template was successfully created, issTempID: ", issTempIDx, "cveNum: ", cve.CveNum) // Create issue comment affectedVersion := "" if len(brandArray) > 0 { @@ -224,7 +224,7 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st affectedVersion = affectedVersion + "\n" } errx := CreateIssueComment(accessToken, owner, path, assignee, cve, resp, affectedVersion) - logs.Info("issue评论创建结果, err: ", errx) + logs.Info("Issue comment creation result, err: ", errx) // Send gitee private message content := fmt.Sprintf("%v 仓库有新的CVE和安全问题的ISSUE被 cve-manager 创建,CVE编号: %v, 请及时处理.", path, cve.CveNum) SendPrivateLetters(accessToken, content, assignee) @@ -232,10 +232,10 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st issueID := int64(resp["id"].(float64)) err = CreateDepositHooks(accessToken, owner, path, cve, issueNum, issueID) if err != nil { - logs.Error("创建hooks 失败, cveNum: ", cve.CveNum, "err: ", err) + logs.Error("Failed to create hooks, cveNum: ", cve.CveNum, "err: ", err) return "", err } - logs.Info("创建hooks 成功, cveNum: ", cve.CveNum) + logs.Info("Create hooks successfully, cveNum: ", cve.CveNum) // Update issue status models.UpdateIssueStatus(cve, 2) // Update score status @@ -246,7 +246,7 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st models.DeleteIssueTemplate(issTempID) } } else { - logs.Error("重复创建issue, cve: ", cve, ", templateErr: ", templateErr) + logs.Error("Repeat issue, cve: ", cve, ", templateErr: ", templateErr) return "", nil } } @@ -267,10 +267,10 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st CreateSecNoticeData(&sec, cve, path, branchs, sc.NVDScore) secID, noticeErr := models.UpdateSecNotice(&sec) if noticeErr != nil { - logs.Error("更新安全信息失败,CveNum: ", cve.CveNum, "path: ", path, "err: ", noticeErr) + logs.Error("Failed to update security information,CveNum: ", cve.CveNum, "path: ", path, "err: ", noticeErr) return "", nil } - logs.Info("更新安全信息成功, secID: ", secID, "cveNum: ", cve.CveNum) + logs.Info("Update security information successfully, secID: ", secID, "cveNum: ", cve.CveNum) return "", nil } @@ -393,12 +393,13 @@ func CreateDepositHooks(accessToken string, owner string, path string, hookurl := BConfig.String("hook::hookurl") ihs, errh := models.GetIssueHook(&ih) if errh { - var hookExist = false + hookExist := false for _, wh := range ihs { if wh.HookUrl == hookurl { hookExist = true + hookId := strconv.FormatInt(wh.HookId,10) if accessToken != "" && owner != "" && path != "" { - url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/hooks/" + strconv.Itoa(int(wh.HookId)) + url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/hooks/" + hookId push_events := "true" tag_push_events := "true" issues_events := "true" @@ -599,8 +600,8 @@ func CreateSecNoticeData(sec *models.SecurityNotice, iss models.VulnCenter, sec.CveNum = iss.CveNum opScoreLeve := openEulerScoreProc(opScore) sec.Introduction = "An update for " + path + " is now available for " + branchs + "." - sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ";\n\n" + "openEuler Security has rated this" + - " update as having a security impact of " + opScoreLeve + ". A Common Vunlnerability" + + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openEuler Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + " Scoring System(CVSS)base score,which gives a detailed severity rating," + " is available for each vulnerability from the CVElink(s) in the References section." sec.AffectProduct = branchs diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 3da91d4044e6588c07506cacf043398bb0aafe0f..31aa16058afc0d457d3ef023d728a254b3257027 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -819,7 +819,7 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) return true, nil } -func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum int) (bool, error) { +func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYears int) (bool, error) { if cveData.CveNum == "" || len(cveData.CveNum) == 0 { logs.Error("当前数据cveNum 为空暂不处理,data: ", cveData) models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) @@ -827,9 +827,21 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum int) ( } god, ok := models.QueryCveOpeneulerdata(cveData.PackName, cveData.CveVersion) if !ok { - logs.Error("仓库和版本对应错误, data: ", cveData) + logs.Error("The repo and version correspond to errors, data: ", cveData) models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 4) - return false, errors.New("仓库和版本对应错误,暂时不处理") + return false, errors.New("The repo and version correspond to errors and will not be processed temporarily") + } + // Import cve as data after 2018 + cveNumList := strings.Split(cveData.CveNum, "-") + if cveNumList != nil && len(cveNumList) > 1 { + cveYears, yearErr := strconv.Atoi(cveNumList[1]) + if yearErr == nil { + if cveYears <= manYears { + logs.Error("limited cve years, cveData: ", cveData, ", err: ", cveYears, manYears) + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 7) + return false, errors.New("limited cve years") + } + } } goe, ok := models.QueryCveOpeneulerDetaildataByName(cveData.PackName, cveData.CveVersion) if !ok { @@ -876,6 +888,12 @@ func GetCveOriginData(prcnum, days, openeulernum int, cveRef string) (bool, erro for i, cveOrg := range osx { count = count + 1 logs.Info("当前正常解析第： ", count, "条数据,i:", i, ", cvenum: ", cveOrg.Ids) + // Add mutex + lockOk := models.LockUpdateOriginStatus(common.GetCurTime(), cveOrg.PackName, cveOrg.Version, cveOrg.CveId, 15) + if !lockOk { + logs.Error("Current data is being processed: PackName: ", cveOrg.PackName, cveOrg.Version, cveOrg.CveId) + continue + } go func(idx int, cveData models.OriginUpstream) { ok, err := GenCveVuler(cveData, cveRef, openeulernum) if !ok { @@ -894,7 +912,11 @@ func GetCveOriginData(prcnum, days, openeulernum int, cveRef string) (bool, erro func GetCveOriginExcelData(prcnum, days, openeulerNum int, cveRef string) (bool, error) { defer common.Catchs() - //var os []models.OriginUpstream + manYears, ok := beego.AppConfig.Int("cve::cve_number_m") + if ok != nil { + logs.Error("config cve::manYears error:", ok) + return false, ok + } count := 0 beforeTime := common.GetBeforeTime(days) for { @@ -910,13 +932,19 @@ func GetCveOriginExcelData(prcnum, days, openeulerNum int, cveRef string) (bool, for i, cveOrg := range osx { count = count + 1 logs.Info("当前正常解析第： ", count, "条数据,i:", i, ", cvenum: ", cveOrg.CveNum) - go func(idx int, cveData models.OriginExcel) { - ok, err := SyncCveVuler(cveData, cveRef, openeulerNum) + // add mutex + lockOk := models.LockUpdateOriginExcelStatus(common.GetCurTime(), cveOrg.CveId, 15) + if !lockOk { + logs.Error("Current data is being processed, cveNum: ", cveOrg.CveNum, ",cveId: ", cveOrg.CveId) + continue + } + go func(idx, manYears int, cveData models.OriginExcel) { + ok, err := SyncCveVuler(cveData, cveRef, openeulerNum, manYears) if !ok { - logs.Error("SyncCveVuler, cveData: ", cveData, "处理失败, err: ", err) + logs.Error("SyncCveVuler, 处理失败, err: ", err, "cveData: ", cveData) } ch <- i - }(i, cveOrg) + }(i, manYears, cveOrg) } for i := 0; i < cap(ch); i++ { <-ch @@ -972,8 +1000,8 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c affectBrands = lop.InfProduct } sec.Introduction = "An update for " + lop.Components + " is now available for " + affectBrands + "." - sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ";\n\n" + "openEuler Security has rated this" + - " update as having a security impact of " + opScoreLeve + ". A Common Vunlnerability" + + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openEuler Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + " Scoring System(CVSS)base score,which gives a detailed severity rating," + " is available for each vulnerability from the CVElink(s) in the References section." sec.AffectProduct = affectBrands @@ -1154,8 +1182,8 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c affectBrands = lop.InfProduct } sec.Introduction = "An update for " + lop.Components + " is now available for " + affectBrands + "." - sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ";\n\n" + "openEuler Security has rated this" + - " update as having a security impact of " + opScoreLeve + ". A Common Vunlnerability" + + sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "openEuler Security has rated this" + + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + " Scoring System(CVSS)base score,which gives a detailed severity rating," + " is available for each vulnerability from the CVElink(s) in the References section." sec.AffectProduct = affectBrands @@ -1302,8 +1330,8 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c } var issueTemp models.IssueTemplate issueTemp.CveId = vul.CveId - _, okl := models.GetIssueTemplet(&issueTemp) - if okl && issueTemp.TemplateId > 0 { + templateErr := models.GetIssueTemplateByColName(&issueTemp, "cve_id") + if templateErr == nil && issueTemp.TemplateId > 0 { issueTemp.CveNum = cveData.CveNumber issueTemp.OwnedComponent = lop.Components issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) @@ -1535,12 +1563,24 @@ func GenCveVulerByIssue(cveData models.GiteOriginIssue, cveRef string, openeuler } } else { - lock.Lock() - okx, errx := InsertIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, owner) - lock.Unlock() - if !okx { - logs.Error("数据处理失败， InsertIssueCveGroups， err: ", errx) - return false, err + if strings.ToLower(cveData.State) == "open" || cveData.State == "待办的" || + strings.ToLower(cveData.State) == "progressing" || cveData.State == "进行中" || + strings.ToLower(cveData.State) == "started" || cveData.State == "开启的" { + lock.Lock() + okx, errx := InsertIssueCveGroups(cveData, hole, cveRef, openeulernum, 1, goe, owner) + lock.Unlock() + if !okx { + logs.Error("数据处理失败， InsertIssueCveGroups， err: ", errx) + return false, err + } + } else { + lock.Lock() + okx, errx := InsertIssueCveGroups(cveData, hole, cveRef, openeulernum, 2, goe, owner) + lock.Unlock() + if !okx { + logs.Error("数据处理失败， InsertIssueCveGroups， err: ", errx) + return false, err + } } } models.UpdateCveIssueStatusById(2, cveData.Id) @@ -1563,6 +1603,12 @@ func GetCveIssueData(prcnum, days, openeulernum int, cveRef, owner string) (bool for i, cveOrg := range gs { count = count + 1 logs.Info("当前正常解析第： ", count, "条数据,i:", i, ", cvenum: ", cveOrg.IssueId) + // add mutex + lockOk := models.LockUpdateCveIssueStatus(15, cveOrg.Id) + if !lockOk { + logs.Error("Data is being processed, id: ", cveOrg.Id, ",cveOrg: ", cveOrg) + continue + } if cveOrg.CveNumber == "" || len(cveOrg.CveNumber) == 0 { logs.Error("数据异常, 不处理, cveData: ", cveOrg) models.UpdateCveIssueStatusById(3, cveOrg.Id) @@ -1682,6 +1728,8 @@ func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { if err != nil { logs.Error(err) } + //fileDir := beego.AppConfig.DefaultString("fileDir", "download") + filePath := beego.AppConfig.String("xml::updateinfo_path") if affectBranch == "openEuler-20.03-LTS" { fr := models.ExportRecord{FileName: fileName} err = fr.Read("file_name") @@ -1693,7 +1741,7 @@ func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { du := "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" //du := beego.AppConfig.String("excel::v_pack_20_03_url") du = beego.AppConfig.DefaultString("excel::v_pack_20_03_url", du) - localPath := filepath.Join(dir, affectBranch + "_release-package.CSV") + localPath := filepath.Join(dir, affectBranch+"_release-package.CSV") err = downloadPackageFile(localPath, du) if err != nil { logs.Error(err) @@ -1720,6 +1768,25 @@ func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { } else { logs.Error("affectBranch: ", affectBranch, ", 不能导出sa") } + // zip file + zipFileList := []string{fileName, filePath} + //zipFileList := []string{"download/cve与安全公告openEuler-20.03-LTS_1607081986.xlsx", "download/updateinfo.xml"} + if isExist := fileExist(zipFileList); isExist { + zipFileName := "openEuler_" + common.GetCurDate() + "_" + strconv.Itoa(time.Now().Nanosecond()) + ".zip" + zipFileName = filepath.Join(dir, zipFileName) + zipErr := ZipFiles(zipFileName, zipFileList, dir, dir) + if zipErr != nil { + logs.Error("File compression failed: err: ", zipErr) + } + // send email + sendError := SendEmail(zipFileName) + if sendError != nil { + logs.Error("SendEmail, sendErr: ", sendError) + } + zipFileList = append(zipFileList, zipFileName) + } + // Clear file + delFile(zipFileList) logs.Info(affectBranch, ", GenerateExcelTrigger: end") } @@ -1748,5 +1815,4 @@ func downloadPackageFile(localPath, url string) error { } else { return errors.New("download file request fail") } - } diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 8fb1bbacb12fb76ed9f9cafa02ee38ca8aea18be..795aefc748bc68da2a02da7c30b99f7f3da85753 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -386,6 +386,12 @@ func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, if pl%10 > 0 { pc++ } + var cvexml []CveXml + var dpdates Updates + securityNotice := make(map[string][]SecurityNoticeXml) + packRpmx := make(map[string][]PackRpm) + filePath := beego.AppConfig.String("xml::updateinfo_path") + ReadXml(filePath, &dpdates) cd := make(chan []IssueAndPkg) start := 0 end := 0 @@ -400,12 +406,37 @@ func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, } for i := 0; i < pc; i++ { wgTrigger.Add(1) - go ec.handleGiteData(cd, affectBranch) + go ec.handleGiteData(cd, affectBranch, &cvexml, &dpdates, securityNotice, packRpmx) } wgTrigger.Wait() + // write xml + WriteXml(filePath, cvexml, &dpdates, securityNotice, packRpmx) logs.Info("cve_gen_finish") } +func fileExist(fileList []string) bool { + if len(fileList) > 0 { + for _, filex := range fileList { + _, isErr := os.Stat(filex) + if isErr != nil { + return false + } + } + } + return true +} + +func delFile(fileList []string) { + if len(fileList) > 0 { + for _, filex := range fileList { + err := os.Remove(filex) + if err != nil { + logs.Error(err) + } + } + } +} + func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { list, err := models.GetCanExportCveData(off, size) if err != nil { @@ -462,26 +493,13 @@ func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { return nil } -func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBranch string) (err error) { +func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, + affectBranch string, cvexml *[]CveXml, dpdates *Updates, + securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) (err error) { lz := len(list) if lz > 0 { for _, v := range list { - affectBool := false - if v.AffectProduct != "" && len(v.AffectProduct) > 1 { - if v.AffectProduct != affectBranch { - affectBranchsxList := strings.Split(v.AffectProduct, "/") - for _, ab := range affectBranchsxList { - if ab == affectBranch { - affectBool = true - v.Introduction = strings.ReplaceAll(v.Introduction, v.AffectProduct, affectBranch) - v.Theme = strings.ReplaceAll(v.Theme, v.AffectProduct, affectBranch) - v.AffectProduct = affectBranch - } - } - } else { - affectBool = true - } - } + affectBool := affectBrachRep(&v, affectBranch) if !affectBool { logs.Error("Unaffected version, data: ", v) continue @@ -490,6 +508,9 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran fillLock.Lock() ec.setContentRow(v) fillLock.Unlock() + fillLock.Lock() + BuildXml(cvexml, &v, securityNotice, packRpmx) + fillLock.Unlock() } else if v.Num > 1 { //处理CVE_NUM 重复 //1.根据cve_num 获取issue_tpl 如果所有的issue_status ==2 0r issue_status == 6 则可以导出数据 @@ -508,7 +529,6 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran if t.IssueStatus == 2 { mergerList = append(mergerList, strconv.FormatInt(t.CveId, 10)) } - } //2.从issue_status == 2 的模板数据中找到评分最高的对应的ExcelExport数据 if canMerger && len(mergerList) > 0 { @@ -516,6 +536,9 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran if err != nil { logs.Error("GetCanExportCveDataSameNum, err: ", err) } + // Fill in the content of the xml file + addXmlData(canExport, cvexml, affectBranch, securityNotice, packRpmx) + // Fill in the content of the excel file ep := canExport[0] ep.SecID = v.SecID if ep.AffectProduct != "" && len(ep.AffectProduct) > 1 { @@ -530,24 +553,9 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran m := make(map[string]struct{}) m[ep.OwnedComponent] = struct{}{} for _, ex := range canExport { - affectBool := false - if ex.AffectProduct != "" && len(ex.AffectProduct) > 1 { - if ex.AffectProduct != affectBranch { - affectBranchsxList := strings.Split(ex.AffectProduct, "/") - for _, ab := range affectBranchsxList { - if ab == affectBranch { - affectBool = true - ex.Introduction = strings.ReplaceAll(ex.Introduction, ex.AffectProduct, affectBranch) - ex.Theme = strings.ReplaceAll(ex.Theme, ex.AffectProduct, affectBranch) - ex.AffectProduct = affectBranch - } - } - } else { - affectBool = true - } - } + affectBool := affectBrachRep(&ex, affectBranch) if !affectBool { - logs.Error("Unaffected version, data: ", v) + logs.Error("Unaffected version, data: ", ex) continue } //component repeat do not append @@ -573,6 +581,40 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBran return nil } +func affectBrachRep(xmlp *models.ExcelExport, affectBranch string) bool { + affectBool := false + if xmlp.AffectProduct != "" && len(xmlp.AffectProduct) > 1 { + if xmlp.AffectProduct != affectBranch { + affectBranchsxList := strings.Split(xmlp.AffectProduct, "/") + for _, ab := range affectBranchsxList { + if ab == affectBranch { + affectBool = true + xmlp.Introduction = strings.ReplaceAll(xmlp.Introduction, xmlp.AffectProduct, affectBranch) + xmlp.Theme = strings.ReplaceAll(xmlp.Theme, xmlp.AffectProduct, affectBranch) + xmlp.AffectProduct = affectBranch + } + } + } else { + affectBool = true + } + } + return affectBool +} + +func addXmlData(canExport []models.ExcelExport, cvexml *[]CveXml, affectBranch string, + securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) { + for _, xmlp := range canExport { + affectBool := affectBrachRep(&xmlp, affectBranch) + if !affectBool { + logs.Error("Unaffected version, data: ", xmlp) + continue + } + fillLock.Lock() + BuildXml(cvexml, &xmlp, securityNotice, packRpmx) + fillLock.Unlock() + } +} + func (ec *CveExcel) setContentRow(v models.ExcelExport) { pkg, pkgErr := models.GetCvePackageList(v.SecID) if pkgErr != nil { @@ -800,7 +842,8 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I c <- chData } -func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string) { +func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, cvexml *[]CveXml, + dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) { defer wgTrigger.Done() data := <-c var pkgList []string @@ -829,7 +872,7 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string) logs.Error("GetCanExportExcelData, err: ", err) return } - err = ec.handleWriteContentSync(el, affectBranch) + err = ec.handleWriteContentSync(el, affectBranch, cvexml, dpdates, securityNotice, packRpmx) if err != nil { logs.Error("handleWriteContentSync, err: ", err) } @@ -911,4 +954,3 @@ func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, } return } - diff --git a/taskhandler/hook.go b/taskhandler/hook.go new file mode 100644 index 0000000000000000000000000000000000000000..6c64d3feef7a07c967d2e2f9b4e6bb7d3dc94fda --- /dev/null +++ b/taskhandler/hook.go @@ -0,0 +1,77 @@ +package taskhandler + +import ( + "cvevulner/models" + "cvevulner/util" + "fmt" + "github.com/astaxie/beego/logs" +) + +type HookData struct { + Id int64 + HookUrl string + CreateTime string + Password string + ResultCode int + ProjectId int64 +} + +func ProcHookEvent(hookurl, owner, accessToken, pwd string) error { + hookId := int64(0) + count := 5 + for ; ; { + iHook, ihOk := models.GetAllIssueHook(hookId, count, 1) + if ihOk && len(iHook) > 0 { + for _, ih := range iHook { + var hd []HookData + GetDepositHooks(accessToken, ih.Owner, ih.Repo, &hd) + if len(hd) > 0 { + PrcMutDepositHooks(accessToken, pwd, ih, hd) + } + } + hookId = iHook[len(iHook)-1].Id + } else { + break + } + } + return nil +} + +func GetDepositHooks(accessToken, owner, repo string, hd *[]HookData) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/hooks?access_token=%v&page=1&per_page=50", owner, repo, accessToken) + hookData, err := util.HTTPGet(url) + if err == nil && hookData != nil { + for _, value := range hookData { + if _, ok := value["id"]; !ok { + logs.Error("hookData, err: ", ok, "url: ", url) + continue + } + var hdl HookData + hdl.Id = int64(value["id"].(float64)) + hdl.HookUrl = value["url"].(string) + hdl.CreateTime = value["created_at"].(string) + hdl.Password = value["password"].(string) + hdl.ProjectId = int64(value["project_id"].(float64)) + hdl.ResultCode = int(value["result_code"].(float64)) + *hd = append(*hd, hdl) + } + } +} + +func PrcMutDepositHooks(accessToken, pwd string, ihk models.IssueHooks, hd []HookData) { + for _, hdk := range hd { + if ihk.HookUrl == hdk.HookUrl && hdk.Password == pwd { + if hdk.Id != ihk.HookId && ihk.HookId > 0 { + go DeleteDepositHooks(accessToken, ihk.Owner, ihk.Repo, hdk.Id) + } + } + } +} + +func DeleteDepositHooks(accessToken, owner, repo string, hookId int64) { + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/hooks/%v", owner, repo, hookId) + requestBody := fmt.Sprintf(`{ + "access_token": "%s" + }`, accessToken) + util.HTTPDelCom(url, requestBody) +} diff --git a/taskhandler/oricvecheck.go b/taskhandler/oricvecheck.go index 98facf952ff949199d7468bc44f63ab64e31634c..c072d93b57fa4db4ba137ca586bdc812d65501e3 100644 --- a/taskhandler/oricvecheck.go +++ b/taskhandler/oricvecheck.go @@ -60,3 +60,7 @@ func CheckCveOriginData(prcnum int) (string, error) { } return "", nil } + +func UnLockTableData() { + +} diff --git a/taskhandler/sendemail.go b/taskhandler/sendemail.go new file mode 100644 index 0000000000000000000000000000000000000000..e7de9538d10f2a4e08ba82e8d59683e30544f4dd --- /dev/null +++ b/taskhandler/sendemail.go @@ -0,0 +1,192 @@ +package taskhandler + +import ( + "archive/zip" + "bytes" + "cvevulner/models" + "encoding/base64" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "io" + "io/ioutil" + "net/smtp" + "os" + "strings" + "time" +) + +// define email interface, and implemented auth and send method +type Mail interface { + Auth() + Send(message Message) error +} + +type SendMail struct { + user string + password string + host string + port string + auth smtp.Auth +} + +type Attachment struct { + name string + contentType string + withFile bool +} + +type Message struct { + from string + to []string + cc []string + bcc []string + subject string + body string + contentType string + attachment Attachment +} + +// srcFile could be a single file or a directory +// ZipFiles compresses one or many files into a single zip archive file. +func ZipFiles(filename string, files []string, oldform, newform string) error { + newZipFile, err := os.Create(filename) + if err != nil { + return err + } + defer newZipFile.Close() + zipWriter := zip.NewWriter(newZipFile) + defer zipWriter.Close() + // 把files添加到zip中 + for _, file := range files { + zipfile, err := os.Open(file) + if err != nil { + return err + } + defer zipfile.Close() + // 获取file的基础信息 + info, err := zipfile.Stat() + if err != nil { + return err + } + header, err := zip.FileInfoHeader(info) + if err != nil { + return err + } + //使用上面的FileInforHeader() 就可以把文件保存的路径替换成我们自己想要的了，如下面 + header.Name = strings.Replace(file, oldform, newform, -1) + // 优化压缩 + // 更多参考see http://golang.org/pkg/archive/zip/#pkg-constants + header.Method = zip.Deflate + writer, err := zipWriter.CreateHeader(header) + if err != nil { + return err + } + if _, err = io.Copy(writer, zipfile); err != nil { + return err + } + } + return nil +} + + +func SendEmail(attchStr string) error { + var mail Mail + emailName := beego.AppConfig.String("email::email_name") + emailPwd := beego.AppConfig.String("email::email_pwd") + emailHost := beego.AppConfig.String("email::email_host") + emailPort := beego.AppConfig.String("email::email_port") + elt := models.EmailList{EmailType: 1} + el, eltErr := elt.Read("EmailType") + if eltErr != nil { + logs.Error("Failed to get mailing list, err: ", eltErr) + return eltErr + } + toEmailName := []string{} + for _, em := range el { + toEmailName = append(toEmailName, em.EmailName) + } + mail = &SendMail{user: emailName, password: emailPwd, host: emailHost, port: emailPort} + message := Message{from: emailName, + to: toEmailName, + cc: []string{}, + bcc: []string{}, + subject: "updateinfo.xml and the excel file corresponding to SA", + body: "hi all: \n The attachment is: updateinfo.xml and the excel file corresponding to SA.", + contentType: "text/plain;charset=utf-8", + attachment: Attachment{ + name: attchStr, + contentType: "text/plain", + withFile: true, + }, + } + emailError := mail.Send(message) + return emailError +} + +func (mail *SendMail) Auth() { + mail.auth = smtp.PlainAuth("", mail.user, mail.password, mail.host) +} + +func (mail SendMail) Send(message Message) error { + mail.Auth() + buffer := bytes.NewBuffer(nil) + boundary := "GoBoundary" + Header := make(map[string]string) + Header["From"] = message.from + Header["To"] = strings.Join(message.to, ";") + Header["Cc"] = strings.Join(message.cc, ";") + Header["Bcc"] = strings.Join(message.bcc, ";") + Header["Subject"] = message.subject + Header["Content-Type"] = "multipart/mixed;boundary=" + boundary + Header["Mime-Version"] = "1.0" + Header["Date"] = time.Now().String() + mail.writeHeader(buffer, Header) + body := "\r\n--" + boundary + "\r\n" + body += "Content-Type:" + message.contentType + "\r\n" + body += "\r\n" + message.body + "\r\n" + buffer.WriteString(body) + if message.attachment.withFile { + attachment := "\r\n--" + boundary + "\r\n" + attachment += "Content-Transfer-Encoding:base64\r\n" + attachment += "Content-Disposition:attachment\r\n" + attachment += "Content-Type:" + message.attachment.contentType + ";name=\"" + message.attachment.name + "\"\r\n" + buffer.WriteString(attachment) + defer func() { + if err := recover(); err != nil { + logs.Error(err) + } + }() + mail.writeFile(buffer, message.attachment.name) + } + buffer.WriteString("\r\n--" + boundary + "--") + header := smtp.SendMail(mail.host+":"+mail.port, mail.auth, message.from, message.to, buffer.Bytes()) + logs.Info("header: ", header) + return nil +} + +func (mail SendMail) writeHeader(buffer *bytes.Buffer, Header map[string]string) string { + header := "" + for key, value := range Header { + header += key + ":" + value + "\r\n" + } + header += "\r\n" + buffer.WriteString(header) + return header +} + +// read and write the file to buffer +func (mail SendMail) writeFile(buffer *bytes.Buffer, fileName string) { + file, err := ioutil.ReadFile(fileName) + if err != nil { + panic(err.Error()) + } + payload := make([]byte, base64.StdEncoding.EncodedLen(len(file))) + base64.StdEncoding.Encode(payload, file) + buffer.WriteString("\r\n") + for index, line := 0, len(payload); index < line; index++ { + buffer.WriteByte(payload[index]) + if (index+1)%76 == 0 { + buffer.WriteString("\r\n") + } + } +} \ No newline at end of file diff --git a/taskhandler/xml.go b/taskhandler/xml.go new file mode 100644 index 0000000000000000000000000000000000000000..7a64082a62bdd24829261cf832cbc43d84839dc7 --- /dev/null +++ b/taskhandler/xml.go @@ -0,0 +1,248 @@ +package taskhandler + +import ( + "cvevulner/models" + "encoding/xml" + "fmt" + "github.com/astaxie/beego/logs" + "io/ioutil" + "os" + "strings" + "time" +) + +type Updates struct { + XMLName xml.Name `xml:"updates,omitempty"` + Updatex []Update `xml:"update,omitempty"` +} + +type Update struct { + XMLName xml.Name `xml:"update,omitempty"` + From string `xml:"from,attr"` + Type string `xml:"type,attr"` + Status string `xml:"status,attr"` + Id string `xml:"id"` + Title string `xml:"title"` + Severity string `xml:"severity"` + Release string `xml:"release"` + Issued *Issued `xml:"issued,omitempty"` + References *References `xml:"references,omitempty"` + Description string `xml:"description"` + Pkglist *Pkglist `xml:"pkglist,omitempty"` +} + +type Issued struct { + XMLName xml.Name `xml:"issued,omitempty"` + Date string `xml:"date,attr"` +} + +type References struct { + XMLName xml.Name `xml:"references,omitempty"` + Reference []Reference `xml:"reference,omitempty"` +} + +type Reference struct { + XMLName xml.Name `xml:"reference,omitempty"` + Href string `xml:"href,attr"` + Id string `xml:"id,attr"` + Title string `xml:"title,attr"` + Type string `xml:"type,attr"` +} + +type Pkglist struct { + XMLName xml.Name `xml:"pkglist,omitempty"` + Collection *Collection `xml:"collection,omitempty"` +} + +type Collection struct { + XMLName xml.Name `xml:"collection,omitempty"` + Name string `xml:"name"` + Package []Package `xml:"package,omitempty"` +} + +type Package struct { + XMLName xml.Name `xml:"package,omitempty"` + Arch string `xml:"arch,attr"` + Name string `xml:"name,attr"` + Release string `xml:"release,attr"` + Version string `xml:"version,attr"` + Filename string `xml:"filename"` +} + +// Intermediate information +type SecurityNoticeXml struct { + CveNum string + Title string + Type string + ReferenceLink string + Description string +} + +type PackRpm struct { + PackName string +} + +type CveXml struct { + PublicDate string + OpenEulerSANum string + Introduction string + CveLevel string + OwnedComponent string + SecurityNotice map[string][]SecurityNoticeXml + PackRpmx map[string][]PackRpm +} + +func ReadXml(filePath string, dpdates *Updates) error { + if filePath == "" || len(filePath) == 0 { + filePath = "download/updateinfo.xml" + } + fd, err := os.Open(filePath) + if err != nil { + logs.Error("open file err : ", err, ",filePath: ", filePath) + return err + } + defer fd.Close() + fileContent, err := ioutil.ReadAll(fd) + if err != nil { + logs.Error("read file err : ", err, ", filePath: ", filePath) + return err + } + err = xml.Unmarshal(fileContent, dpdates) + if err != nil { + logs.Error("unmarshal err : ", err, ", fileContent: ", fileContent) + return err + } + return nil +} + +func WriteXml(filePath string, cveXmlList []CveXml, dpdates *Updates, + securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) { + for _, cveXml := range cveXmlList { + upDatex := Update{From: "openeuler.org", Type: "security", Status: "stable", Id: cveXml.OpenEulerSANum, + Title: cveXml.Introduction, Severity: cveXml.CveLevel, Release: "openEuler"} + var pubDate Issued + pubDate.Date = cveXml.PublicDate + upDatex.Issued = &pubDate + var rfs References + if sn, ok := securityNotice[cveXml.OwnedComponent]; ok { + if len(sn) > 0 { + descriptionsList := []string{} + for _, sec := range sn { + var rf Reference + rf.Id = sec.CveNum + rf.Title = sec.CveNum + rf.Href = sec.ReferenceLink + rf.Type = "cve" + descriptionsList = append(descriptionsList, sec.Description) + rfs.Reference = append(rfs.Reference, rf) + } + upDatex.Description = strings.Join(descriptionsList, " ") + } + } + upDatex.References = &rfs + var pl Pkglist + var ct Collection + ct.Name = "openEuler" + if prx, ok := packRpmx[cveXml.OwnedComponent]; ok{ + if len(prx) > 0 { + for _, pr := range prx { + var pe Package + pe.Filename = pr.PackName + //pe.Name = cveXml.OwnedComponent + if len(pr.PackName) > len(cveXml.OwnedComponent) { + //packVersion := pr.PackName[len(cveXml.OwnedComponent) + 1: len(pr.PackName) - 4] + packVersionList := strings.Split(pr.PackName, "-") + if len(packVersionList) >= 3 { + pe.Version = packVersionList[len(packVersionList) - 2] + rpmName := packVersionList[len(packVersionList) - 1][:len(packVersionList[len(packVersionList) - 1]) - 4] + lastIndex := strings.LastIndexAny(rpmName, ".") + if lastIndex != -1 { + pe.Release = rpmName[:lastIndex] + pe.Arch = rpmName[lastIndex + 1:] + } + pe.Name = strings.Join(packVersionList[0:len(packVersionList) - 2], "-") + } + } + ct.Package = append(ct.Package, pe) + } + } + } + pl.Collection = &ct + upDatex.Pkglist = &pl + dpdates.Updatex = append(dpdates.Updatex, upDatex) + } + xmlOutPut, outPutErr := xml.MarshalIndent(dpdates, "", "") + if outPutErr == nil { + //加入XML头 + headerBytes := []byte(xml.Header) + //拼接XML头和实际XML内容 + xmlOutPutData := append(headerBytes, xmlOutPut...) + //写入文件 + ioutil.WriteFile(filePath, xmlOutPutData, os.ModeAppend) + } else { + fmt.Println(outPutErr) + } +} + +func BuildXml(cveXml *[]CveXml, v *models.ExcelExport, + securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) { + pkg, pkgErr := models.GetCvePackageList(v.SecID) + if pkgErr != nil { + logs.Error(pkgErr) + return + } + //cvex.SecurityNotice = make(map[string][]SecurityNoticeXml) + sn := SecurityNoticeXml{CveNum: v.CveNum, Title: v.CveNum, Type: "cve", ReferenceLink: v.ReferenceLink, + Description: deleteTailBlank(v.CveBrief) + "(" + v.CveNum + ")"} + if vx, ok := securityNotice[v.OwnedComponent]; !ok { + snArry := []SecurityNoticeXml{} + snArry = append(snArry, sn) + securityNotice[v.OwnedComponent] = snArry + var cvex CveXml + if v.PublicDate == "" { + v.PublicDate = time.Now().Format("2006-01-02") + } + cvex.OpenEulerSANum = v.OpenEulerSANum + cvex.PublicDate = v.PublicDate + if strings.ToLower(v.CveLevel) == "low" { + cvex.CveLevel = "Low" + } else if strings.ToLower(v.CveLevel) == "medium" { + cvex.CveLevel = "Moderate" + } else if strings.ToLower(v.CveLevel) == "high" { + cvex.CveLevel = "Important" + } else { + cvex.CveLevel = "Critical" + } + cvex.Introduction = v.Introduction + cvex.OwnedComponent = v.OwnedComponent + *cveXml = append(*cveXml, cvex) + } else { + vx = append(vx, sn) + securityNotice[v.OwnedComponent] = vx + } + //cvex.PackRpmx = make(map[string][]PackRpm) + if px, ok := packRpmx[v.OwnedComponent]; !ok { + prArry := []PackRpm{} + for _, v := range pkg { + pr := PackRpm{PackName: v.PackName} + prArry = append(prArry, pr) + } + packRpmx[v.OwnedComponent] = prArry + } else { + for _, v := range pkg { + pnFlag := false + for _, p := range px { + if p.PackName == v.PackName { + pnFlag = true + break + } + } + if !pnFlag { + pr := PackRpm{PackName: v.PackName} + px = append(px, pr) + } + } + packRpmx[v.OwnedComponent] = px + } + return +} diff --git a/util/http.go b/util/http.go index 345c28807e387cd638d6741a3a3b3d56dca3960b..c94762e3bbd0600ba8df4e9b48d653207fa4327d 100644 --- a/util/http.go +++ b/util/http.go @@ -31,7 +31,7 @@ func HTTPPatch(url string, requestBody string) (map[string]interface{}, error) { client := &http.Client{} resp, err := client.Do(req) if err != nil { - logs.Error("Post request 失败, err: ", err, "body: ", requestBody, "url:", url) + logs.Error("PATCH request failed, err: ", err, "body: ", requestBody, "url:", url) return nil, err } defer resp.Body.Close() @@ -39,16 +39,16 @@ func HTTPPatch(url string, requestBody string) (map[string]interface{}, error) { logs.Info("response Headers:", resp.Header, "url: ", url) status, _ := strconv.Atoi(resp.Status) if status > 300 { - logs.Error("Patch request 失败, err: ", err, "body: ", requestBody, "url:", url) + logs.Error("Patch request failed, err: ", err, "body: ", requestBody, "url:", url) return nil, err } body, err := ioutil.ReadAll(resp.Body) fmt.Println("response Body:", string(body)) if err != nil || body == nil{ - logs.Error("post 返回失败, err: ", err, "body: ", requestBody) + logs.Error("PATCH failed, err: ", err, "body: ", requestBody) return nil, err } - logs.Info("post 返回成功!, body: ", string(body)) + logs.Info("PATCH successed!, body: ", string(body)) var iss map[string]interface{} err = json.Unmarshal(body, &iss) if err != nil { @@ -70,7 +70,7 @@ func HTTPPost(url string, requestBody string) (map[string]interface{}, error) { client := &http.Client{} resp, err := client.Do(req) if err != nil { - logs.Error("Post request 失败, err: ", err, "body: ", requestBody, "url:", url) + logs.Error("Post request failed, err: ", err, "body: ", requestBody, "url:", url) return nil, err } defer resp.Body.Close() @@ -78,16 +78,16 @@ func HTTPPost(url string, requestBody string) (map[string]interface{}, error) { logs.Info("response Headers:", resp.Header, "url: ", url) status, _ := strconv.Atoi(resp.Status) if status > 300 { - logs.Error("Post request 失败, err: ", err, "body: ", requestBody, "url:", url) + logs.Error("Post request failed, err: ", err, "body: ", requestBody, "url:", url) return nil, err } body, err := ioutil.ReadAll(resp.Body) fmt.Println("response Body:", string(body)) if err != nil || body == nil { - logs.Error("post 返回失败, err: ", err, "body: ", requestBody) + logs.Error("post failed, err: ", err, "body: ", requestBody) return nil, err } - logs.Info("post 返回成功!, body: ", string(body)) + logs.Info("post successed!, body: ", string(body)) var iss map[string]interface{} err = json.Unmarshal(body, &iss) if err != nil { @@ -109,7 +109,7 @@ func HTTPPost1(url string, requestBody string) ([]map[string]interface{}, error) client := &http.Client{} resp, err := client.Do(req) if err != nil { - logs.Error("Post request 失败, err: ", err, "body: ", requestBody, "url:", url) + logs.Error("Post request failed, err: ", err, "body: ", requestBody, "url:", url) return nil, err } defer resp.Body.Close() @@ -117,16 +117,16 @@ func HTTPPost1(url string, requestBody string) ([]map[string]interface{}, error) logs.Info("response Headers:", resp.Header, "url: ", url) status, _ := strconv.Atoi(resp.Status) if status > 300 { - logs.Error("Post request 失败, err: ", err, "body: ", requestBody, "url:", url) + logs.Error("Post request failed, err: ", err, "body: ", requestBody, "url:", url) return nil, err } body, err := ioutil.ReadAll(resp.Body) fmt.Println("response Body:", string(body)) if err != nil || body == nil{ - logs.Error("post 返回失败, err: ", err, "body: ", requestBody) + logs.Error("post failed, err: ", err, "body: ", requestBody) return nil, err } - logs.Info("post 返回成功!, body: ", string(body)) + logs.Info("post successed!, body: ", string(body)) var iss []map[string]interface{} err = json.Unmarshal(body, &iss) if err != nil { @@ -210,3 +210,42 @@ func HTTPGetCom(urls string) ([]byte, error) { logs.Info("body: \n", string(body), "url: ", urls) return body, nil } + +func HTTPDelCom(url, requestBody string) (map[string]interface{}, error) { + req, err := http.NewRequest("DELETE", url, bytes.NewBuffer([]byte(requestBody))) + defer common.Catchs() + if err != nil { + logs.Error("NewRequest, url: ", url, ",req: ", requestBody, ",err: ", err) + return nil, err + } + req.Header.Set("Content-Type", "application/json") + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + logs.Error("DELETE request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + defer resp.Body.Close() + logs.Info("response Status:", resp.Status, "url: ", url) + logs.Info("response Headers:", resp.Header, "url: ", url) + status, _ := strconv.Atoi(resp.Status) + if status > 300 { + logs.Error("DELETE request failed, err: ", err, "body: ", requestBody, "url:", url) + return nil, err + } + body, err := ioutil.ReadAll(resp.Body) + fmt.Println("response Body:", string(body)) + if err != nil || body == nil{ + logs.Error("DELETE failed, err: ", err, "body: ", requestBody) + return nil, err + } + logs.Info("DELETE successed!, body: ", string(body)) + var iss map[string]interface{} + err = json.Unmarshal(body, &iss) + if err != nil { + logs.Error(err, string(body)) + return nil, err + } + logs.Info(iss) + return iss, nil +}