From 87ad9b9b57cf87361c7ebfbdba9227b9c43c03ad Mon Sep 17 00:00:00 2001 From: zhangjianjun Date: Tue, 30 Nov 2021 11:33:00 +0800 Subject: [PATCH 1/2] 1. Submit an issue manually, and the robot submits it again, because the real-time query code cloud data failed to return; --- cve-vulner-manager/controllers/hook.go | 6 +- .../cve-py/controller/timertaskcontroller.py | 2 +- cve-vulner-manager/cve-py/main.py | 16 ++++- cve-vulner-manager/main.go | 1 + cve-vulner-manager/models/cve.go | 7 ++ cve-vulner-manager/models/modeldb.go | 12 ++++ cve-vulner-manager/taskhandler/createissue.go | 67 +++++++++++-------- 7 files changed, 77 insertions(+), 34 deletions(-) diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index 9368cd7..b894453 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -2898,6 +2898,10 @@ func DelOrgIssue(issueHook *models.IssuePayload) { organizationID := int8(1) organizationID = taskhandler.GetOrganizationId(nameSpace) if issueType == CIssueType || strings.HasPrefix(issueTitle, "CVE") { + // Data deletion record + idr := models.IssueDeleteRecord{IssueId: issueHook.Issue.Id, IssueNum: issueNumber, + DelAssignee: issueHook.Sender.Login, Owner: nameSpace, Repo: repoPath, DeleteTime: common.GetCurTime()} + models.InsertIssueDeleteRecord(&idr) item := models.GiteOriginIssue{IssueId: issueHook.Issue.Id, Url: issueHook.Issue.HtmlUrl, Number: issueNumber, State: issueState, Title: issueTitle, IssueType: issueType, SecurityHole: true, @@ -2909,7 +2913,7 @@ func DelOrgIssue(issueHook *models.IssuePayload) { if strings.HasPrefix(issueTitle, "CVE") { item.CveNumber = issueTitle } else if issueHook.Issue.Body != "" { - //通过正则表达式去body中截取 稍后完善 + // Use regular expressions to intercept the body and improve it later sm := util.RegexpCveNumber.FindAllStringSubmatch(issueHook.Issue.Body, -1) if len(sm) > 0 && len(sm[0]) > 0 { item.CveNumber = util.TrimString(sm[0][1]) diff --git a/cve-vulner-manager/cve-py/controller/timertaskcontroller.py b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py index 565725f..6404323 100644 --- a/cve-vulner-manager/cve-py/controller/timertaskcontroller.py +++ b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py @@ -23,7 +23,7 @@ def timertask(): :return:None """ try: - scheduler = background.BackgroundScheduler(job_defaults={'max_instances': 100}) + scheduler = background.BackgroundScheduler(job_defaults={'max_instances': 200}) # Add tasks # Warehouse, organization, and owner correspondence timing task scheduler.add_job(taskcontroller.gwcontroller, 'cron', day_of_week='0-6', hour=0, minute=2) diff --git a/cve-vulner-manager/cve-py/main.py b/cve-vulner-manager/cve-py/main.py index 9bcf633..c1ec139 100644 --- a/cve-vulner-manager/cve-py/main.py +++ b/cve-vulner-manager/cve-py/main.py @@ -20,6 +20,8 @@ from flask import request from controller import timertaskcontroller from tabletask import supplement_cve from gevent import pywsgi +from threading import Thread + app = flask.Flask(__name__) @@ -45,9 +47,17 @@ def pull_cve_info(): } +def api_proc(): + server = pywsgi.WSGIServer(('0.0.0.0', 8080), app) + server.serve_forever() + if __name__ == '__main__': print("Start a scheduled task...") - timertaskcontroller.timertask() - server = pywsgi.WSGIServer(('0.0.0.0', 8080), app) - server.serve_forever() + thread_timed_task = Thread(target=timertaskcontroller.timertask) + thread_timed_task.start() + thread_timed_task.join() + thread_api_proc = Thread(target=api_proc) + thread_api_proc.start() + thread_api_proc.join() + diff --git a/cve-vulner-manager/main.go b/cve-vulner-manager/main.go index 57e1b4e..2606737 100644 --- a/cve-vulner-manager/main.go +++ b/cve-vulner-manager/main.go @@ -32,6 +32,7 @@ func main() { } // single run task.StartTask() + defer task.StopTask() if beego.BConfig.RunMode == "dev" { beego.BConfig.WebConfig.DirectoryIndex = true beego.BConfig.WebConfig.StaticDir["/swagger"] = "swagger" diff --git a/cve-vulner-manager/models/cve.go b/cve-vulner-manager/models/cve.go index a4fcd02..570a544 100644 --- a/cve-vulner-manager/models/cve.go +++ b/cve-vulner-manager/models/cve.go @@ -851,6 +851,13 @@ func GetSaNumber(sa *SaNumber, field ...string) error { return err } +func InsertIssueDeleteRecord(idr *IssueDeleteRecord) error { + o := orm.NewOrm() + id, err := o.Insert(idr) + logs.Info("InsertIssueDeleteRecord, id: ", id, ", err: ", err) + return err +} + func InsertSaNumber(sa *SaNumber) error { o := orm.NewOrm() id, err := o.Insert(sa) diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index 89a3a91..cbd20bb 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -980,6 +980,17 @@ type CommunityYamlConfig struct { OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore"` } +type IssueDeleteRecord struct { + IssueDeleteId int64 `orm:"pk;auto;column(id)"` + IssueId int64 `orm:"column(issue_id)" description:"issue的id"` + IssueNum string `orm:"size(64);column(issue_num)" description:"issue编号"` + DelAssignee string `orm:"size(128);column(issue_assignee)" description:"issue所属责任人"` + IssueLabel string `orm:"size(256);column(issue_label)" description:"issue标签, CVE/FIXED, CVE/UNFIXED"` + Owner string `orm:"size(128);column(owner)" description:"仓库地址"` + Repo string `orm:"size(512);column(repo)" description:"仓库路径"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + func CreateDb() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -1020,6 +1031,7 @@ func CreateDb() bool { new(MindSporeBrandTags), new(OriginUpstreamRecord), new(OpenLookengSecurityReviewer), new(OpenLookengYaml), new(IssueCommunityStatistics), new(CommunityYamlConfig), + new(IssueDeleteRecord), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index fa4179e..7cd7428 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -178,23 +178,28 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, } } assigneeGite := "" - if it.TemplateId > 0 && len(it.IssueNum) > 2 { - issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, it.IssueNum) - if issueErr != nil { - models.DeleteIssueTemplate(it.TemplateId) - models.UpdateIssueStatus(cve, 0) - return "", errors.New("Recreate issue") - } else { - if issueBody != nil && issueBody["assignee"] != nil { - assigneeObj := issueBody["assignee"].(map[string]interface{}) - if assigneeObj != nil && assigneeObj["login"] != nil { - assigneeGite = assigneeObj["login"].(string) + if cve.DataSource == 4 && cve.CveId == it.CveId && len(it.IssueNum) > 2 { + logs.Info("Current data is synchronized") + } else { + if it.TemplateId > 0 && len(it.IssueNum) > 2 { + issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, it.IssueNum) + if issueErr != nil { + models.DeleteIssueTemplate(it.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } else { + if issueBody != nil && issueBody["assignee"] != nil { + assigneeObj := issueBody["assignee"].(map[string]interface{}) + if assigneeObj != nil && assigneeObj["login"] != nil { + assigneeGite = assigneeObj["login"].(string) + } } } + } else { + logs.Error("CreateIssueToGit, GetIssueTemplateByColName, templateErr: ", templateErr, ",it: ", it) } - } else { - logs.Error("CreateIssueToGit, GetIssueTemplateByColName, templateErr: ", templateErr, ",it: ", it) } + if it.TemplateId > 0 && len(it.IssueNum) > 2 { if it.Assignee == "" || len(it.Assignee) == 0 { it.Assignee = assignee @@ -464,26 +469,30 @@ func UpdateIssueToGit(accessToken string, owner string, path string, accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") } assigneeGite := "" - if its.IssueNum != "" && len(its.IssueNum) > 2 { - issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, its.IssueNum) - if issueErr != nil { - models.DeleteIssueTemplate(its.TemplateId) - models.UpdateIssueStatus(cve, 0) - return "", errors.New("Recreate issue") - } else { - if issueBody != nil { - if issueBody != nil && issueBody["assignee"] != nil { - assigneeObj := issueBody["assignee"].(map[string]interface{}) - if assigneeObj != nil && assigneeObj["login"] != nil { - assigneeGite = assigneeObj["login"].(string) + if cve.DataSource == 4 && cve.CveId == its.CveId && len(its.IssueNum) > 2 { + logs.Info("Current data is synchronized") + } else { + if its.IssueNum != "" && len(its.IssueNum) > 2 { + issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, its.IssueNum) + if issueErr != nil { + models.DeleteIssueTemplate(its.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } else { + if issueBody != nil { + if issueBody != nil && issueBody["assignee"] != nil { + assigneeObj := issueBody["assignee"].(map[string]interface{}) + if assigneeObj != nil && assigneeObj["login"] != nil { + assigneeGite = assigneeObj["login"].(string) + } } } } + } else { + models.DeleteIssueTemplate(its.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") } - } else { - models.DeleteIssueTemplate(its.TemplateId) - models.UpdateIssueStatus(cve, 0) - return "", errors.New("Recreate issue") } //labels := its.IssueLabel if cve.OrganizationID == 1 { -- Gitee From c10f8ff5e44b6e9da2e687459961e3fca3a61a32 Mon Sep 17 00:00:00 2001 From: zhangjianjun Date: Tue, 30 Nov 2021 11:59:39 +0800 Subject: [PATCH 2/2] Optimized format --- cve-vulner-manager/cve-py/main.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cve-vulner-manager/cve-py/main.py b/cve-vulner-manager/cve-py/main.py index c1ec139..111cb0e 100644 --- a/cve-vulner-manager/cve-py/main.py +++ b/cve-vulner-manager/cve-py/main.py @@ -20,7 +20,7 @@ from flask import request from controller import timertaskcontroller from tabletask import supplement_cve from gevent import pywsgi -from threading import Thread +import threading app = flask.Flask(__name__) @@ -48,16 +48,19 @@ def pull_cve_info(): def api_proc(): + """ + 1. api thread processing function + """ server = pywsgi.WSGIServer(('0.0.0.0', 8080), app) server.serve_forever() if __name__ == '__main__': print("Start a scheduled task...") - thread_timed_task = Thread(target=timertaskcontroller.timertask) + thread_timed_task = threading.Thread(target=timertaskcontroller.timertask) thread_timed_task.start() thread_timed_task.join() - thread_api_proc = Thread(target=api_proc) + thread_api_proc = threading.Thread(target=api_proc) thread_api_proc.start() thread_api_proc.join() -- Gitee