diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index 9368cd7a7691ca326de942a8de2e1254db7eee5c..b8944539f23021580614df0d0ddf2f4e1e4810de 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -2898,6 +2898,10 @@ func DelOrgIssue(issueHook *models.IssuePayload) { organizationID := int8(1) organizationID = taskhandler.GetOrganizationId(nameSpace) if issueType == CIssueType || strings.HasPrefix(issueTitle, "CVE") { + // Data deletion record + idr := models.IssueDeleteRecord{IssueId: issueHook.Issue.Id, IssueNum: issueNumber, + DelAssignee: issueHook.Sender.Login, Owner: nameSpace, Repo: repoPath, DeleteTime: common.GetCurTime()} + models.InsertIssueDeleteRecord(&idr) item := models.GiteOriginIssue{IssueId: issueHook.Issue.Id, Url: issueHook.Issue.HtmlUrl, Number: issueNumber, State: issueState, Title: issueTitle, IssueType: issueType, SecurityHole: true, @@ -2909,7 +2913,7 @@ func DelOrgIssue(issueHook *models.IssuePayload) { if strings.HasPrefix(issueTitle, "CVE") { item.CveNumber = issueTitle } else if issueHook.Issue.Body != "" { - //通过正则表达式去body中截取 稍后完善 + // Use regular expressions to intercept the body and improve it later sm := util.RegexpCveNumber.FindAllStringSubmatch(issueHook.Issue.Body, -1) if len(sm) > 0 && len(sm[0]) > 0 { item.CveNumber = util.TrimString(sm[0][1]) diff --git a/cve-vulner-manager/cve-py/controller/timertaskcontroller.py b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py index 565725fe55e6f4b47c38a04e356a141960c88c08..64043236ca8eab4000d20e47828318cef79062c2 100644 --- a/cve-vulner-manager/cve-py/controller/timertaskcontroller.py +++ b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py @@ -23,7 +23,7 @@ def timertask(): :return:None """ try: - scheduler = background.BackgroundScheduler(job_defaults={'max_instances': 100}) + scheduler = background.BackgroundScheduler(job_defaults={'max_instances': 200}) # Add tasks # Warehouse, organization, and owner correspondence timing task scheduler.add_job(taskcontroller.gwcontroller, 'cron', day_of_week='0-6', hour=0, minute=2) diff --git a/cve-vulner-manager/cve-py/main.py b/cve-vulner-manager/cve-py/main.py index 9bcf633a4eaa24bf7d176da4940a23d99d731c00..111cb0ef51d6f14c791bac1388d5299a5f2c4710 100644 --- a/cve-vulner-manager/cve-py/main.py +++ b/cve-vulner-manager/cve-py/main.py @@ -20,6 +20,8 @@ from flask import request from controller import timertaskcontroller from tabletask import supplement_cve from gevent import pywsgi +import threading + app = flask.Flask(__name__) @@ -45,9 +47,20 @@ def pull_cve_info(): } +def api_proc(): + """ + 1. api thread processing function + """ + server = pywsgi.WSGIServer(('0.0.0.0', 8080), app) + server.serve_forever() + if __name__ == '__main__': print("Start a scheduled task...") - timertaskcontroller.timertask() - server = pywsgi.WSGIServer(('0.0.0.0', 8080), app) - server.serve_forever() + thread_timed_task = threading.Thread(target=timertaskcontroller.timertask) + thread_timed_task.start() + thread_timed_task.join() + thread_api_proc = threading.Thread(target=api_proc) + thread_api_proc.start() + thread_api_proc.join() + diff --git a/cve-vulner-manager/main.go b/cve-vulner-manager/main.go index 57e1b4e8035aa87403c783364467eed5527171e4..2606737e5f24e8d72c632cc18535daec72d8955d 100644 --- a/cve-vulner-manager/main.go +++ b/cve-vulner-manager/main.go @@ -32,6 +32,7 @@ func main() { } // single run task.StartTask() + defer task.StopTask() if beego.BConfig.RunMode == "dev" { beego.BConfig.WebConfig.DirectoryIndex = true beego.BConfig.WebConfig.StaticDir["/swagger"] = "swagger" diff --git a/cve-vulner-manager/models/cve.go b/cve-vulner-manager/models/cve.go index a4fcd02e12bc357b29f0fb9b7e154055b8de763a..570a544e403dd952e8ee1e0d9f037e7d3c607e2c 100644 --- a/cve-vulner-manager/models/cve.go +++ b/cve-vulner-manager/models/cve.go @@ -851,6 +851,13 @@ func GetSaNumber(sa *SaNumber, field ...string) error { return err } +func InsertIssueDeleteRecord(idr *IssueDeleteRecord) error { + o := orm.NewOrm() + id, err := o.Insert(idr) + logs.Info("InsertIssueDeleteRecord, id: ", id, ", err: ", err) + return err +} + func InsertSaNumber(sa *SaNumber) error { o := orm.NewOrm() id, err := o.Insert(sa) diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index 89a3a91508b41656bf107003670ec13f0ac7c3a8..cbd20bb1b69ca78632222c63c43fab759e52021e 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -980,6 +980,17 @@ type CommunityYamlConfig struct { OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore"` } +type IssueDeleteRecord struct { + IssueDeleteId int64 `orm:"pk;auto;column(id)"` + IssueId int64 `orm:"column(issue_id)" description:"issue的id"` + IssueNum string `orm:"size(64);column(issue_num)" description:"issue编号"` + DelAssignee string `orm:"size(128);column(issue_assignee)" description:"issue所属责任人"` + IssueLabel string `orm:"size(256);column(issue_label)" description:"issue标签, CVE/FIXED, CVE/UNFIXED"` + Owner string `orm:"size(128);column(owner)" description:"仓库地址"` + Repo string `orm:"size(512);column(repo)" description:"仓库路径"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + func CreateDb() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -1020,6 +1031,7 @@ func CreateDb() bool { new(MindSporeBrandTags), new(OriginUpstreamRecord), new(OpenLookengSecurityReviewer), new(OpenLookengYaml), new(IssueCommunityStatistics), new(CommunityYamlConfig), + new(IssueDeleteRecord), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index fa4179e13c87b0cc1ec9c4c613bae5e020f62054..7cd7428ffac43519489bfcafb33cf832e47a102e 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -178,23 +178,28 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, } } assigneeGite := "" - if it.TemplateId > 0 && len(it.IssueNum) > 2 { - issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, it.IssueNum) - if issueErr != nil { - models.DeleteIssueTemplate(it.TemplateId) - models.UpdateIssueStatus(cve, 0) - return "", errors.New("Recreate issue") - } else { - if issueBody != nil && issueBody["assignee"] != nil { - assigneeObj := issueBody["assignee"].(map[string]interface{}) - if assigneeObj != nil && assigneeObj["login"] != nil { - assigneeGite = assigneeObj["login"].(string) + if cve.DataSource == 4 && cve.CveId == it.CveId && len(it.IssueNum) > 2 { + logs.Info("Current data is synchronized") + } else { + if it.TemplateId > 0 && len(it.IssueNum) > 2 { + issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, it.IssueNum) + if issueErr != nil { + models.DeleteIssueTemplate(it.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } else { + if issueBody != nil && issueBody["assignee"] != nil { + assigneeObj := issueBody["assignee"].(map[string]interface{}) + if assigneeObj != nil && assigneeObj["login"] != nil { + assigneeGite = assigneeObj["login"].(string) + } } } + } else { + logs.Error("CreateIssueToGit, GetIssueTemplateByColName, templateErr: ", templateErr, ",it: ", it) } - } else { - logs.Error("CreateIssueToGit, GetIssueTemplateByColName, templateErr: ", templateErr, ",it: ", it) } + if it.TemplateId > 0 && len(it.IssueNum) > 2 { if it.Assignee == "" || len(it.Assignee) == 0 { it.Assignee = assignee @@ -464,26 +469,30 @@ func UpdateIssueToGit(accessToken string, owner string, path string, accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") } assigneeGite := "" - if its.IssueNum != "" && len(its.IssueNum) > 2 { - issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, its.IssueNum) - if issueErr != nil { - models.DeleteIssueTemplate(its.TemplateId) - models.UpdateIssueStatus(cve, 0) - return "", errors.New("Recreate issue") - } else { - if issueBody != nil { - if issueBody != nil && issueBody["assignee"] != nil { - assigneeObj := issueBody["assignee"].(map[string]interface{}) - if assigneeObj != nil && assigneeObj["login"] != nil { - assigneeGite = assigneeObj["login"].(string) + if cve.DataSource == 4 && cve.CveId == its.CveId && len(its.IssueNum) > 2 { + logs.Info("Current data is synchronized") + } else { + if its.IssueNum != "" && len(its.IssueNum) > 2 { + issueErr, issueBody := GetGiteeIssue(accessToken, owner, path, its.IssueNum) + if issueErr != nil { + models.DeleteIssueTemplate(its.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") + } else { + if issueBody != nil { + if issueBody != nil && issueBody["assignee"] != nil { + assigneeObj := issueBody["assignee"].(map[string]interface{}) + if assigneeObj != nil && assigneeObj["login"] != nil { + assigneeGite = assigneeObj["login"].(string) + } } } } + } else { + models.DeleteIssueTemplate(its.TemplateId) + models.UpdateIssueStatus(cve, 0) + return "", errors.New("Recreate issue") } - } else { - models.DeleteIssueTemplate(its.TemplateId) - models.UpdateIssueStatus(cve, 0) - return "", errors.New("Recreate issue") } //labels := its.IssueLabel if cve.OrganizationID == 1 {