diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index 145343e74ca5902be3441e8d5b19aa59d86535e2..29b48293fd9ef224aa73a3dbc88c8ef1dcd0b588 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -330,7 +330,7 @@ func getMaintainer(path, prSender, assignee string) string { func gaussCloseIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, token, owner, fixed, unFix, path string, cveCenter *models.VulnCenter) { - unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.OwnedComponent, + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, issueTmp.OwnedVersion, cveCenter.OrganizationID) if len(unFixList) > 0 { //send comment to issue @@ -392,7 +392,7 @@ func gaussCloseIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueT func sporeCloseIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, token, owner, fixed, unFix, path string, cveCenter *models.VulnCenter) { - unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.OwnedComponent, + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, issueTmp.OwnedVersion, cveCenter.OrganizationID) if len(unFixList) > 0 { //send comment to issue @@ -473,7 +473,7 @@ func closeIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTempla sigReviewSend(issueHook, issueTmp, token, owner, fixed, unFix, assignee, cveCenter) } if openScoreFlag { - unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.OwnedComponent, + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, issueTmp.OwnedVersion, cveCenter.OrganizationID) if len(unFixList) > 0 { //send comment to issue @@ -556,31 +556,33 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { unFix := beego.AppConfig.String("labelUnFix") fixed := beego.AppConfig.String("labelFixed") uNaffected := beego.AppConfig.String("labeUnaffected") - gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") issueId := issueHook.Issue.Id issueTmp := models.IssueTemplate{} issueTmp.IssueId = issueId issueTmp.IssueNum = issueHook.Iid + repoPath := "" if issueHook.Issue.Repository.Path != "" && - len(issueHook.Issue.Repository.Path) > 1 && - issueHook.Issue.Repository.Path != gaussIssuePath { - issueTmp.OwnedComponent = issueHook.Issue.Repository.Path - issueErr := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "owned_component", "issue_id") - if issueErr != nil { - return issueErr - } - } else { - issueErr := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") - if issueErr != nil { - return issueErr - } + len(issueHook.Issue.Repository.Path) > 1 { + repoPath = issueHook.Issue.Repository.Path } + issueErr := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") + if issueErr != nil { + return issueErr + } + path := issueTmp.Repo cveCenter := models.VulnCenter{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum} cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") if cveErr != nil { return cveErr } - path := issueTmp.Repo + if len(repoPath) > 1 && repoPath != path { + cveCenter.PackName = repoPath + cveCenter.RepoName = issueTmp.OwnedComponent + models.UpdateVulnCenter(&cveCenter, "PackName", "RepoName") + issueTmp.Repo = repoPath + models.UpdateIssueTemplate(&issueTmp, "Repo") + path = repoPath + } token := beego.AppConfig.String("gitee::git_token") owner := beego.AppConfig.String("gitee::owner") if cveCenter.OrganizationID == 2 { @@ -588,26 +590,12 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") owner = gaussOwner token = gitGaussToken - gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") - path = gaussIssuePath } else if cveCenter.OrganizationID == 3 { mindsporeOwner := beego.AppConfig.String("mindspore::mindspore_owner") gitMindsporeToken := beego.AppConfig.String("mindspore::git_mindspore_token") owner = mindsporeOwner token = gitMindsporeToken // Query the repo that needs to submit an issue - cveList := strings.Split(cveCenter.CveVersion, ",") - if len(cveList) > 0 { - for _, cl := range cveList { - ms := models.MindSporeYaml{PackageName: cveCenter.PackName, Version: cl} - sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") - if ms.Id > 0 { - path = ms.Repo - break - } - logs.Info("GetMindSporeYaml, sporeErr: ", sporeErr) - } - } } issueTmp.StatusName = issueHook.Issue.StateName logs.Info("Initiating issue status modification, sponsor: @", issueHook.Sender.UserName, ", Modify status: ", @@ -685,24 +673,12 @@ func VerifyIssueAsPr(issueTmp *models.IssueTemplate, cveCenter models.VulnCenter affectedBranchs := "" token := "" owner := "" - path := "" + path := cveCenter.PackName if cveCenter.OrganizationID == 3 { affectedBranchs = beego.AppConfig.String("mindspore::mindspore_version") owner = beego.AppConfig.String("mindspore::mindspore_owner") token = beego.AppConfig.String("mindspore::git_mindspore_token") // Query the repo that needs to submit an issue - cveList := strings.Split(cveCenter.CveVersion, ",") - if len(cveList) > 0 { - for _, cl := range cveList { - ms := models.MindSporeYaml{PackageName: cveCenter.PackName, Version: cl} - sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") - if ms.Id > 0 { - path = ms.Repo - break - } - logs.Info("GetMindSporeYaml, sporeErr: ", sporeErr) - } - } if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { tmpTagList := make([]string, 0) affectProductSlice := strings.Split(sn.AffectProduct, "/") @@ -724,10 +700,10 @@ func VerifyIssueAsPr(issueTmp *models.IssueTemplate, cveCenter models.VulnCenter affectedBranchs = beego.AppConfig.String("opengauss::gauss_version") token = beego.AppConfig.String("opengauss::git_gauss_token") owner = beego.AppConfig.String("opengauss::gauss_owner") - issuePath := beego.AppConfig.String("opengauss::gauss_issue_path") - //prRepo := beego.AppConfig.String("opengauss::pr_repo") - //prRepoSlice = strings.Split(prRepo, ",") - path = issuePath + if len(path) < 2 { + issuePath := beego.AppConfig.String("opengauss::gauss_issue_path") + path = issuePath + } } else { affectedBranchs = beego.AppConfig.String("cve::affected_branchs") token = beego.AppConfig.String("gitee::git_token") @@ -1092,26 +1068,7 @@ func updateTempAndCenter(issueTmp models.IssueTemplate, cveCenter models.VulnCen labelFixed := beego.AppConfig.String("labelFixed") labelUnFix := beego.AppConfig.String("labelUnFix") labeAbiChanged := beego.AppConfig.String("labeAbiChanged") - path := "" - if cveCenter.OrganizationID == 3 { - // Query the repo that needs to submit an issue - cveList := strings.Split(cveCenter.CveVersion, ",") - if len(cveList) > 0 { - for _, cl := range cveList { - ms := models.MindSporeYaml{PackageName: cveCenter.PackName, Version: cl} - sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") - if ms.Id > 0 { - path = ms.Repo - break - } - logs.Info("GetMindSporeYaml, sporeErr: ", sporeErr) - } - } - } else if cveCenter.OrganizationID == 2 { - path = beego.AppConfig.String("opengauss::gauss_issue_path") - } else { - path = issueTmp.Repo - } + path := cveCenter.PackName sn := models.SecurityNotice{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum} snErr := sn.Read("cve_id", "cve_num") if snErr != nil { @@ -1284,7 +1241,7 @@ func openEulerScoreReview(issueTmp *models.IssueTemplate, cuAccount, owner, toke func gaussMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, unfixed, path string, cveCenter models.VulnCenter) { - unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.OwnedComponent, + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, issueTmp.OwnedVersion, cveCenter.OrganizationID) if len(unFixList) > 0 { na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" @@ -1345,7 +1302,7 @@ func gaussMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, to func sporeMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, unfixed, path string, cveCenter models.VulnCenter) { - unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.OwnedComponent, + unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, issueTmp.Repo, issueTmp.OwnedVersion, cveCenter.OrganizationID) if len(unFixList) > 0 { na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" @@ -1404,7 +1361,7 @@ func sporeMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, to func maintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, unfixed string, organizationID int8) { unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, - issueTmp.OwnedComponent, issueTmp.OwnedVersion, organizationID) + issueTmp.Repo, issueTmp.OwnedVersion, organizationID) if len(unFixList) > 0 { na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" cc := fmt.Sprintf(CommentCheckVersion, cuAccount, strings.Join(unFixList, ",")) + na @@ -1467,7 +1424,7 @@ func maintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, func securityApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, unfixed string, organizationID int8) { unFixList := taskhandler.CheckAffectVerComplete(issueTmp.AffectedVersion, - issueTmp.OwnedComponent, issueTmp.OwnedVersion, organizationID) + issueTmp.Repo, issueTmp.OwnedVersion, organizationID) if len(unFixList) > 0 { na := "\n**请确认分支信息是否填写完整,否则将无法关闭当前issue.**" cc := fmt.Sprintf(CommentCheckVersion, cuAccount, strings.Join(unFixList, ",")) + na @@ -1539,6 +1496,7 @@ func handleIssueComment(payload models.CommentPayload) { issueId := payload.Issue.Id // issue id int64 cBody := payload.Comment.Body //Comment subject cuAccount := payload.Comment.User.UserName //gitee domain address + repoPath := payload.Issue.Repository.Path cmdRej := beego.AppConfig.DefaultString("rejectCmd", "/reject") cmdApe := beego.AppConfig.DefaultString("approveCmd", "/approve") cmdClose := beego.AppConfig.DefaultString("closeCmd", "/close") @@ -1583,26 +1541,24 @@ func handleIssueComment(payload models.CommentPayload) { if vc.OrganizationID == 2 { owner = beego.AppConfig.String("opengauss::gauss_owner") accessToken = beego.AppConfig.String("opengauss::git_gauss_token") - path = beego.AppConfig.String("opengauss::gauss_issue_path") + if len(path) < 2 { + path = beego.AppConfig.String("opengauss::gauss_issue_path") + } cBody = strings.ReplaceAll(cBody, util.KwOpenGaussScore, util.KwOpenEulerScore) } else if vc.OrganizationID == 3 { owner = beego.AppConfig.String("mindspore::mindspore_owner") accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") // Query the repo that needs to submit an issue - cveList := strings.Split(vc.CveVersion, ",") - if len(cveList) > 0 { - for _, cl := range cveList { - ms := models.MindSporeYaml{PackageName: vc.PackName, Version: cl} - sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") - if ms.Id > 0 { - path = ms.Repo - break - } - logs.Info("GetMindSporeYaml, sporeErr: ", sporeErr) - } - } cBody = strings.ReplaceAll(cBody, util.KwMindSporeScore, util.KwOpenEulerScore) } + if len(repoPath) > 1 && repoPath != path { + vc.PackName = repoPath + vc.RepoName = issueTmp.OwnedComponent + models.UpdateVulnCenter(&vc, "PackName", "RepoName") + issueTmp.Repo = repoPath + models.UpdateIssueTemplate(&issueTmp, "Repo") + path = repoPath + } if strings.HasPrefix(cBody, cmdRej) { //Review rejected Add comment @Analyst if !isReviewer(cuAccount) { @@ -2916,8 +2872,8 @@ func DelOrgIssue(issueHook *models.IssuePayload) { issueTmp := models.IssueTemplate{} issueTmp.IssueId = issueHook.Issue.Id issueTmp.IssueNum = issueNumber - issueTmp.OwnedComponent = repoPath - issueErr := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "owned_component", "issue_id") + issueTmp.Repo = repoPath + issueErr := models.GetIssueTemplateByColName(&issueTmp, "IssueNum", "Repo", "IssueId") if issueErr != nil { return } @@ -2929,7 +2885,7 @@ func DelOrgIssue(issueHook *models.IssuePayload) { if tpErr != nil { logs.Error(tpErr) } - cveCenter := models.VulnCenter{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum, PackName: issueTmp.OwnedComponent} + cveCenter := models.VulnCenter{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum, PackName: issueTmp.Repo} cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num", "pack_name") if cveErr != nil { return @@ -2998,8 +2954,8 @@ func AddIssueComment(token, owner, path, issueNum, assignee string, issueId int6 issueTmp := models.IssueTemplate{} issueTmp.IssueId = issueId issueTmp.IssueNum = issueNum - issueTmp.OwnedComponent = path - issueErr := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "owned_component", "issue_id") + issueTmp.Repo = path + issueErr := models.GetIssueTemplateByColName(&issueTmp, "IssueNum", "Repo", "IssueId") if issueErr != nil { logs.Error("GetIssueTemplateByColName, err: ", issueErr, ", issue: ", issueTmp) return issueErr diff --git a/cve-vulner-manager/cve-py/common/files.py b/cve-vulner-manager/cve-py/common/files.py index e63ac4a3cc2a4d3090fa9b6ba010b4a049461e86..23c9e7311fba2e8ba25ba7b499e103311ca593a7 100644 --- a/cve-vulner-manager/cve-py/common/files.py +++ b/cve-vulner-manager/cve-py/common/files.py @@ -15,6 +15,7 @@ Date: 04/01/2021 11:01 AM import os import shutil +import yaml def mkdirs(file_path, file_flag): @@ -52,3 +53,13 @@ def file_isexists(file_path): return True else: return False + + +def parse_yaml(file_name): + """ + Parse yaml + return dict + """ + with open(file_name) as f: + yaml_data = yaml.safe_load(f) + return yaml_data diff --git a/cve-vulner-manager/cve-py/controller/taskcontroller.py b/cve-vulner-manager/cve-py/controller/taskcontroller.py index ecf5c8c95631e7de78ed828de1a25f71e166855b..7e89f6b8d6d8cdbde5bf27e7a5328e5024d372c5 100644 --- a/cve-vulner-manager/cve-py/controller/taskcontroller.py +++ b/cve-vulner-manager/cve-py/controller/taskcontroller.py @@ -17,7 +17,8 @@ Date: 10/22/2020 11:01 AM from tabletask import (runtask, mappingtask, toexcel, export_excel_task, import_excel_task, package_committer_task, spec_error_task, repeattask, gauss_yaml, - issue_statistics, supplement_cve, mindspore_yaml) + issue_statistics, supplement_cve, mindspore_yaml, + openlookeng_yaml) from gitwebtask import genegroup, yamltask from emailtask import sendingtask, issue_record_email from deletetask import deletefiletask @@ -180,3 +181,12 @@ def parse_mindspore_yaml_task(): print('the task of Process mindspore yaml data start') mindspore_yaml.proc_mindspore_yaml() print('the task of Process mindspore yaml data complete') + +def parse_openlookeng_yaml_task(): + """ + Download the yaml file first, then parse the yaml file of openLooKeng + Returns:None + """ + print('the task of Process openLooKeng yaml data start') + openlookeng_yaml.proc_openlookeng_yaml() + print('the task of Process openLooKeng yaml data complete') diff --git a/cve-vulner-manager/cve-py/controller/timertaskcontroller.py b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py index dd6b6ded2db4edc88b0f07fa126b4183c48417e2..7ff5ae7628cf5cee3528d3cb0fdc58e2245fa7d4 100644 --- a/cve-vulner-manager/cve-py/controller/timertaskcontroller.py +++ b/cve-vulner-manager/cve-py/controller/timertaskcontroller.py @@ -50,6 +50,8 @@ def timertask(): scheduler.add_job(taskcontroller.long_supplement_cve_task, 'cron', day_of_week='0-6', hour=1, minute=30) # Parse the yaml file of mindspore scheduler.add_job(taskcontroller.parse_mindspore_yaml_task, 'cron', day_of_week='0-6', hour=3, minute=30) + # Parse the yaml file of openLooKeng + # scheduler.add_job(taskcontroller.parse_openlookeng_yaml_task, 'cron', day_of_week='0-6', hour=4, minute=30) scheduler.start() except SystemExit as err: print("Err:", err) diff --git a/cve-vulner-manager/cve-py/tabletask/common.py b/cve-vulner-manager/cve-py/tabletask/common.py index 7ab2b8d410406061ad1e6b62b0e611cf1ee89e6a..5cd09f8ee68499149637d74c95ddcf0fdb06d979 100644 --- a/cve-vulner-manager/cve-py/tabletask/common.py +++ b/cve-vulner-manager/cve-py/tabletask/common.py @@ -78,6 +78,20 @@ def select_gauss_yaml_origin_data(package_name, version, mysql): return None +def select_openlookeng_yaml_origin_data(package_name, version, mysql): + """ + Query origin data + """ + sql = "SELECT id FROM cve_open_lookeng_yaml WHERE package_name = %s " \ + "and version = %s" + val = (package_name, version) + packages_data = mysql.getOne(sql, val) + if packages_data: + return packages_data + else: + return None + + def delete_yaml_data(): """ delete yaml data @@ -91,7 +105,8 @@ def delete_yaml_data(): if pk["package_id"] >= 10000000: gauss_data = select_gauss_yaml_origin_data(pk["package_name"], pk["version"], mysql) spore_data = select_mindspore_yaml_origin_data(pk["package_name"], pk["version"], mysql) - if not gauss_data and not spore_data: + lookeng_data = select_openlookeng_yaml_origin_data(pk["package_name"], pk["version"], mysql) + if not gauss_data and not spore_data and not lookeng_data: print("ID of the currently deleted data: ", pk["git_id"]) delete_yaml_openeuler_detail_data(mysql, pk["git_id"]) delete_yaml_openeuler_data(mysql, pk["git_id"]) diff --git a/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py b/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py index 7b82025ae97a425ef8638d030a3a5ee0b6b2bee0..6ef3bce390a0d1c22c5384465bfaba7772935182 100644 --- a/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py +++ b/cve-vulner-manager/cve-py/tabletask/gauss_yaml.py @@ -17,7 +17,6 @@ from dbConnecttion.MysqlConn import Mysql import pymysql from common import files from common import times -import yaml from tabletask import common from downloadtask import downloadfiletask @@ -40,16 +39,6 @@ def download_gauss_yaml(): return file_name -def parse_yaml(file_name): - """ - Parse yaml - return dict - """ - with open(file_name) as f: - yaml_data = yaml.safe_load(f) - return yaml_data - - def add_package_id(pk_id): """ add index @@ -99,8 +88,8 @@ def store_yaml_data(yaml_data): mysql.dispose() except pymysql.err.IntegrityError: print(pymysql.err.IntegrityError) - # except Exception as e: - # print(e) + # except Exception as e: + # print(e) mysql.dispose(2) mysql.close() @@ -195,9 +184,9 @@ def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): Update origin data """ update_sql = "update cve_open_guss_yaml set origin_url = %s,status = %s, " \ - "cpe_name = %s, update_time = %s where id = %s" + "cpe_name = %s, update_time = %s,repo_name=%s where id = %s" val = (yaml_value["url"], 1, yaml_value["cpeName"], - times.get_current_time(), pg_origin_id) + times.get_current_time(), "security", pg_origin_id) mysql.update(update_sql, val) mysql.dispose() @@ -207,11 +196,11 @@ def insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id): insert origin data """ insert_sql = "insert into cve_open_guss_yaml(package_name," \ - "version,origin_url, status, cpe_name,create_time,update_time, id) " \ - "values(%s,%s,%s,%s,%s,%s,%s,%s)" + "version,origin_url, status, cpe_name,create_time,update_time, id, repo_name) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" val = (yaml_key, yaml_value["version"], yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), - times.get_current_time(), origin_id) + times.get_current_time(), origin_id, "security") last_id = mysql.insertOne(insert_sql, val) mysql.dispose() return last_id @@ -278,7 +267,7 @@ def proc_gauss_yaml(): """ file_name = download_gauss_yaml() if files.file_isexists(file_name): - yaml_data = parse_yaml(file_name) + yaml_data = files.parse_yaml(file_name) if yaml_data is not None and len(yaml_data) > 0: store_yaml_data(yaml_data) else: diff --git a/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py b/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py index 215507ba3ef90f7ee6960918e3c9d49ab937fd0d..6988cd83b970e500e8e23a8b85fc733498ac4053 100644 --- a/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py +++ b/cve-vulner-manager/cve-py/tabletask/mindspore_yaml.py @@ -17,7 +17,6 @@ from dbConnecttion.MysqlConn import Mysql import pymysql from common import files from common import times -import yaml from downloadtask import downloadfiletask from tabletask import common @@ -39,16 +38,6 @@ def download_mindspore_yaml(): return file_name -def parse_yaml(file_name): - """ - Parse yaml - return dict - """ - with open(file_name) as f: - yaml_data = yaml.safe_load(f) - return yaml_data - - def add_package_id(pk_id): """ add index @@ -262,8 +251,8 @@ def store_yaml_data(yaml_data): mysql.dispose() except pymysql.err.IntegrityError: print(pymysql.err.IntegrityError) - # except Exception as e: - # print(e) + # except Exception as e: + # print(e) mysql.dispose(2) mysql.close() @@ -278,7 +267,7 @@ def proc_mindspore_yaml(): """ file_name = download_mindspore_yaml() if files.file_isexists(file_name): - yaml_data = parse_yaml(file_name) + yaml_data = files.parse_yaml(file_name) if yaml_data is not None and len(yaml_data) > 0: store_yaml_data(yaml_data) else: diff --git a/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py b/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py new file mode 100644 index 0000000000000000000000000000000000000000..bd04bf132756c9922dd0a87a458067cf84bde2cb --- /dev/null +++ b/cve-vulner-manager/cve-py/tabletask/openlookeng_yaml.py @@ -0,0 +1,276 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Handle openLooKeng tasks + +Authors: zhangjianjun +Date: 08/25/2021 11:01 AM +""" + +from dbConnecttion.MysqlConn import Mysql +import pymysql +from common import files +from common import times +from downloadtask import downloadfiletask +from tabletask import common + + +def download_openlookeng_yaml(): + """ + Download the yaml file of mindspore on gitee + return file_name + """ + file_path = "./openlookeng_yaml/" + mk_ok = files.mkdirs(file_path, 1) + file_name = file_path + 'openlookeng_yaml.yaml' + if mk_ok: + # download the yaml file + file_url = 'https://gitee.com/mindspore/community/raw/' \ + 'master/security/config/Third_Party_Open_Source_Software_List.yaml' + downloadfiletask.download_yaml(file_name, file_url) + # pass + return file_name + + +def add_package_id(pk_id): + """ + add index + """ + packg_id = 10000000 + if pk_id > 0: + packg_id = pk_id + 1 + return packg_id + + +def select_openeuler_yaml_data(mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id,package_name,version FROM cve_git_open_euler " \ + "WHERE package_id >= %s" + val = (10000000,) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def delete_yaml_openeuler_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_open_euler where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def delete_yaml_openeuler_detail_data(mysql, git_id): + """ + delete origin data + """ + delete_sql = "delete from cve_git_package_info where git_id = %s" + val = (git_id,) + mysql.delete(delete_sql, val) + mysql.dispose() + + +def insert_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + insert data + """ + insert_sql = "insert into cve_git_open_euler(package_id,package_name," \ + "version,origin_url,create_time,update_time,cpe_packname,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" + val = (package_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), yaml_value["cpeName"], 1) + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def insert_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + insert data details + """ + insert_sql = "insert into cve_git_package_info(git_id,package_name," \ + "version,origin_url,create_time,update_time,decription,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" + val = (pg_detail_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), "", 0) + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def update_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + update data + """ + update_sql = "update cve_git_open_euler set origin_url = %s,update_time = %s, " \ + "cpe_packname = %s,status=%s where package_id = %s " \ + "and package_name =%s and version = %s" + val = (yaml_value["url"], times.get_current_time(), yaml_value["cpeName"], + 1, package_id, yaml_key, yaml_value["version"]) + mysql.update(update_sql, val) + + +def update_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + Update data details + """ + update_sql = "update cve_git_package_info set package_name = %s,version = %s, " \ + "origin_url = %s, update_time = %s,status=%s where git_id = %s" + val = (yaml_key, yaml_value["version"], yaml_value["url"], + times.get_current_time(), 0, pg_detail_id) + mysql.update(update_sql, val) + + +def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): + """ + Update origin data + """ + update_sql = "update cve_open_lookeng_yaml set origin_url = %s,status = %s, " \ + "cpe_name = %s, update_time = %s where id = %s" + val = (yaml_value["url"], 1, yaml_value["cpeName"], + times.get_current_time(), pg_origin_id) + mysql.update(update_sql, val) + mysql.dispose() + + +def insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id, repo_key): + """ + insert origin data + """ + insert_sql = "insert into cve_open_lookeng_yaml(package_name," \ + "version,origin_url, status, cpe_name,create_time,update_time, id, repo_name) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" + val = (yaml_key, yaml_value["version"], + yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), + times.get_current_time(), origin_id, repo_key) + last_id = mysql.insertOne(insert_sql, val) + mysql.dispose() + return last_id + + +def select_yaml_data(yaml_key, yaml_value, mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id FROM cve_git_open_euler " \ + "WHERE package_name = %s " \ + "and version = %s order by package_id desc" + val = (yaml_key, yaml_value["version"]) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def delete_yaml_origin_data(mysql): + """ + Update origin data + """ + delete_sql = "delete from cve_open_lookeng_yaml" + mysql.delete(delete_sql) + mysql.dispose() + + +def select_yaml_origin_data(package_name, version, mysql, repo_key): + """ + Query origin data + """ + sql = "SELECT id FROM cve_open_lookeng_yaml WHERE package_name = %s " \ + "and version = %s and repo_name = %s" + val = (package_name, version, repo_key) + packages_data = mysql.getOne(sql, val) + if packages_data: + return packages_data + else: + return None + + +def select_yaml_lastdata(): + """ + Query the last data + """ + mysql = Mysql() + sql = "SELECT package_id FROM cve_git_open_euler order by git_id desc" + package_ids = mysql.getOne(sql) + if package_ids and len(package_ids) > 0: + return package_ids + else: + return None + + +def store_yaml_data(yaml_data): + """ + parse dict + Store yaml data to mysql + return None + """ + mysql = Mysql() + if yaml_data is not None and len(yaml_data) > 0: + # Delete all data before updating + delete_yaml_origin_data(mysql) + origin_id = 1 + for repo_key, repo_value in yaml_data.items(): + try: + for yaml_key, yaml_value in repo_value.items(): + origin_data = select_yaml_origin_data(yaml_key, yaml_value["version"], mysql, repo_key) + if origin_data: + update_yaml_origin_data(origin_data["id"], yaml_value, mysql) + else: + insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id, repo_key) + origin_id += 1 + packages_data = select_yaml_data(yaml_key, yaml_value, mysql) + if packages_data is not None: + if len(packages_data) > 1: + for pd in packages_data[1:]: + delete_yaml_openeuler_detail_data(mysql, pd["git_id"]) + delete_yaml_openeuler_data(mysql, pd["git_id"]) + pk = packages_data[0] + update_yaml_data(pk["package_id"], yaml_key, yaml_value, mysql) + update_yaml_detail_data(pk["git_id"], yaml_key, yaml_value, mysql) + else: + package_ids = select_yaml_lastdata() + if package_ids is not None and package_ids["package_id"] >= 10000000: + packg_id = add_package_id(package_ids["package_id"]) + else: + packg_id = add_package_id(0) + last_id = insert_yaml_data(packg_id, yaml_key, yaml_value, mysql) + insert_yaml_detail_data(last_id, yaml_key, yaml_value, mysql) + mysql.dispose() + except pymysql.err.IntegrityError: + print(pymysql.err.IntegrityError) + # except Exception as e: + # print(e) + mysql.dispose(2) + mysql.close() + + +def proc_openlookeng_yaml(): + """ + 1. Create a folder; + 2. Download yaml; + 3. Parse yaml; + 4. Save in mysql; + return None + """ + file_name = download_openlookeng_yaml() + if files.file_isexists(file_name): + yaml_data = files.parse_yaml(file_name) + if yaml_data is not None and len(yaml_data) > 0: + store_yaml_data(yaml_data) + else: + print("File download failed, file: ", file_name) + # delete history yaml_data + common.delete_yaml_data() diff --git a/cve-vulner-manager/models/excel.go b/cve-vulner-manager/models/excel.go index a6cc1e72ff2810a4f6ded6a65aeba2e11fcd6aeb..8b8fbe625786d54ff21c59495f964b70ec835985 100644 --- a/cve-vulner-manager/models/excel.go +++ b/cve-vulner-manager/models/excel.go @@ -27,6 +27,7 @@ type ExcelExport struct { OrganizateId int8 `json:"organizate_id" orm:"column(organizate_id)"` AffectedVersion string `json:"affected_version" orm:"column(affected_version)"` IssueLabel string `json:"issue_label" orm:"column(issue_label)"` + Repo string `json:"repo" orm:"column(repo)"` } //ExcelPackage Released packages @@ -159,4 +160,4 @@ func GetCvrfSaRecordByCve(cveNum string) (afl []CvrfSaRecord) { logs.Error("cve_cvrf_sa_record, err: ", err) } return afl -} \ No newline at end of file +} diff --git a/cve-vulner-manager/models/gauss.go b/cve-vulner-manager/models/gauss.go index 10c6a02117af7b6a444f1c21d2b56a84bdaaf6ba..045d9448c4dfa44b9877271d7ea86fc2e371f357 100644 --- a/cve-vulner-manager/models/gauss.go +++ b/cve-vulner-manager/models/gauss.go @@ -60,7 +60,7 @@ func QueryGaussExportSaData(cveNum, issueNum, repo string, issueId int64) (list sql := `SELECT b.num,c.*,a.issue_num,a.owned_component,a.cve_brief, d.sec_id,d.introduction,d.summary,d.theme,d.description,d.influence_component, d.affect_product,d.reference_link,d.affect_status, -e.public_date,e.openeuler_sa_num,a.cve_level,b.organizate_id,a.affected_version,a.issue_label +e.public_date,e.openeuler_sa_num,a.cve_level,b.organizate_id,a.affected_version,a.issue_label,a.repo FROM cve_issue_template a RIGHT JOIN (SELECT (SELECT COUNT(*) FROM cve_vuln_center WHERE cve_num = ? AND @@ -124,7 +124,7 @@ func InsertGaussVersion(ogv *OpenGaussVersion) (int64, error) { func GetGaussIssueNumber(packName string) (issueTemp []IssueTemplate, err error) { sql := `select * from cve_issue_template where status = 3 and issue_status = 2 and cve_id in (select cve_id from cve_vuln_center where cve_status = 2 and -is_export in (0,3) and pack_name in ('%s') and organizate_id = 2)` +is_export in (0,3) and repo_name in ('%s') and organizate_id = 2)` sql = fmt.Sprintf(sql, packName) o := orm.NewOrm() _, err = o.Raw(sql).QueryRows(&issueTemp) diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index 43631b407897c77dc0db7c298a4ec62dfd2720b3..998c1c2d8547487d446ef0ceef34136214242868 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -44,7 +44,8 @@ type VulnCenter struct { RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` FirstPerTime string `orm:"size(32);column(first_per_time)" description:"cve首次披露时间"` FirstGetTime string `orm:"size(32);column(first_get_time)" description:"cve受影响后首次感知时间"` - PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称的仓库名称"` + RepoName string `orm:"size(512);column(repo_name)" description:"cve对应得包名称的组件名称"` CveUrl string `orm:"type(text);column(cve_url)" description:"cve下载链接"` IsExport int8 `orm:"default(0);column(is_export)" description:"0:数据初始化;1:数据已同步到官网；2：数据不需要同步到官网；3：issue已关闭需要同步到官网"` DataSource int8 `orm:"default(1);column(data_source)" description:"1:软件所提供;2:网络爬取；3:人工处理; 4:兼容之前的数据"` @@ -142,7 +143,7 @@ type IssueTemplate struct { TemplateId int64 `orm:"pk;auto;column(template_id)"` CveId int64 `orm:"index;column(cve_id)"` CveNum string `orm:"size(256);column(cve_num)" description:"cve编号"` - OwnedComponent string `orm:"size(256);column(owned_component)" description:"漏洞归属组件"` + OwnedComponent string `orm:"size(512);column(owned_component)" description:"漏洞归属组件"` OwnedVersion string `orm:"size(512);column(owned_version)" description:"漏洞归属版本"` NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd评分"` OpenEulerScore float64 `orm:"digits(10);decimals(1);column(openeuler_score)" description:"openeuler评分"` @@ -164,7 +165,7 @@ type IssueTemplate struct { SaAuditFlag int8 `orm:"default(0);column(sa_audit_flag)" description:"pr审核： 0：审核中；1：审核通过；2：审核拒绝"` OpAuditFlag int8 `orm:"default(0);column(op_audit_flag)" description:"评分审核： 0：审核中；1：审核通过；2：审核拒绝"` Owner string `orm:"size(128);column(owner)" description:"仓库地址"` - Repo string `orm:"size(128);column(repo)" description:"仓库路径"` + Repo string `orm:"size(512);column(repo)" description:"仓库路径"` Title string `orm:"size(512);column(title)" description:"issue标题"` IssueType string `orm:"size(64);column(issue_type)" description:"CVE和安全问题"` Collaborators string `orm:"size(128);column(collaborators);null" description:"协助者"` @@ -636,6 +637,7 @@ type IssueCreateRecord struct { type OpenGussYaml struct { Id int64 `orm:"pk;auto"` PackageName string `orm:"column(package_name);size(256)" description:"包名称"` + Repo string `orm:"column(repo_name);size(256)" description:"仓名称"` Version string `orm:"size(64);column(version);index" description:"版本号"` OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` @@ -751,6 +753,7 @@ type OpenGaussCveList struct { CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` + RepoName string `orm:"size(512);column(repo_name)" description:"cve对应得仓名称"` NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd 评分"` OpenEulerScore float64 `orm:"digits(10);decimals(1);column(openeuler_score)" description:"openeuler评分"` NvectorVule string `orm:"size(256);column(n_vector_value)" description:"nvd vector 原始值"` @@ -991,7 +994,7 @@ func CreateDb() bool { new(OpenEulerRepoOrigin), new(OpenGaussSecurityReviewer), new(MindSporeYaml), new(MindSporeSecurityReviewer), new(MindSporeBrandTags), new(OriginUpstreamRecord), - + new(OpenLookengSecurityReviewer), new(OpenLookengYaml), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/cve-vulner-manager/models/spore.go b/cve-vulner-manager/models/spore.go index 9ab89d8eabe76dbc73b568e85d7263856ec95cc2..919ae3f608c71f8b025ac286ff4ab41af2f2e6d3 100644 --- a/cve-vulner-manager/models/spore.go +++ b/cve-vulner-manager/models/spore.go @@ -24,4 +24,12 @@ func QueryMindSporeBrandTags(osl *MindSporeBrandTags, field ...string) error { o := orm.NewOrm() err := o.Read(osl, field...) return err -} \ No newline at end of file +} + +func QueryMindSporeTagsByBrand(osl *MindSporeBrandTags) (mbt []MindSporeBrandTags) { + o := orm.NewOrm() + _, err := o.Raw("SELECT * from cve_mind_spore_brand_tags WHERE package_name = ? and brand = ?", + osl.PackageName, osl.Brand).QueryRows(&mbt) + logs.Info("QueryMindSporeTagsByBrand, err: ", err) + return +} diff --git a/cve-vulner-manager/models/uploadcve.go b/cve-vulner-manager/models/uploadcve.go index 5c17222303dd4dacac8a466bc335881088bf7664..82293953e65407904a6a0ff5f318ba1811909634 100644 --- a/cve-vulner-manager/models/uploadcve.go +++ b/cve-vulner-manager/models/uploadcve.go @@ -975,6 +975,35 @@ func GetOpengaussYaml(opy *OpenGussYaml, colName ...string) error { return err } +func GetOpengaussYamlAll(opy *OpenGussYaml) (msy []OpenGussYaml, err error) { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("GetOpengaussYamlAll, openGauss request query parameters: ", *opy) + return msy, errors.New("openGauss request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 1 { + verStr := "(" + opy.Version + ")" + num, err := o.Raw("select * from cve_open_guss_yaml where package_name = ? and version in ? group by repo_name"+ + "order by id asc", opy.PackageName, verStr).QueryRows(&msy) + if num == 0 { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetOpengaussYamlAll, err: ", err, *opy) + } + return msy, err + } else { + num, err := o.Raw("select * from cve_open_guss_yaml where package_name = ? "+ + "order by id asc", opy.PackageName).QueryRows(&msy) + if num == 0 { + logs.Error("GetOpengaussYamlAll2, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return msy, err + } +} + func GetMindSporeYaml(opy *MindSporeYaml, colName ...string) error { o := orm.NewOrm() packageName := opy.PackageName @@ -1003,6 +1032,35 @@ func GetMindSporeYaml(opy *MindSporeYaml, colName ...string) error { } } +func GetMindSporeYamlAll(opy *MindSporeYaml) (msy []MindSporeYaml, err error) { + o := orm.NewOrm() + packageName := opy.PackageName + if len(opy.PackageName) < 1 && len(opy.Version) < 1 { + logs.Error("GetMindSporeYamlAll, Mindspore request query parameters: ", *opy) + return msy, errors.New("Mindspore request query parameters error") + } + if len(opy.PackageName) > 1 && len(opy.Version) > 1 { + verStr := "(" + opy.Version + ")" + num, err := o.Raw("select * from cve_mind_spore_yaml where package_name = ? and version in ? group by repo_name"+ + "order by id asc", opy.PackageName, verStr).QueryRows(&msy) + if num == 0 { + opy.Repo = packageName + opy.Id = 1 + logs.Error("GetMindSporeYamlAll, err: ", err, *opy) + } + return msy, err + } else { + num, err := o.Raw("select * from cve_mind_spore_yaml where package_name = ? "+ + "order by id asc", opy.PackageName).QueryRows(&msy) + if num == 0 { + logs.Error("GetMindSporeYamlAll2, err: ", err, *opy) + opy.Repo = packageName + opy.Id = 1 + } + return msy, err + } +} + func InsertOriginCveRecord(our *OriginUpstreamRecord) error { o := orm.NewOrm() id, err := o.Insert(our) diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go index 0aca68c502e016ab284059b79631f2e53a80c9e5..70a8af60939ce4b68bafe5858c1d0113b078a380 100644 --- a/cve-vulner-manager/task/issuetask.go +++ b/cve-vulner-manager/task/issuetask.go @@ -177,15 +177,16 @@ func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, fl } else if issueValue.OrganizationID == 2 { owner = beego.AppConfig.String("opengauss::gauss_owner") accessToken = beego.AppConfig.String("opengauss::git_gauss_token") - } - se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} - seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") - if seError == nil && se.Id > 0 { - models.UpdateIssueStatus(issueValue, 5) - logs.Info("addUnlimitedIssue, The current issue does not need to be processed, "+ - "it has been processed, cveData: ", issueValue) - ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) - continue + } else { + se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} + seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") + if seError == nil && se.Id > 0 { + models.UpdateIssueStatus(issueValue, 5) + logs.Info("addUnlimitedIssue, The current issue does not need to be processed, "+ + "it has been processed, cveData: ", issueValue) + ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) + continue + } } // Determine whether the issue has been processed goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName, issueValue.OrganizationID) @@ -221,7 +222,7 @@ func addUnlimitedIssue(beforeTime string, prcnum, years, toolYears, manYears, fl } } } else if issueValue.OrganizationID == 2 { - ogc := models.OpenGaussCveList{CveNum: issueValue.CveNum, PackName: issueValue.PackName, Status: 3} + ogc := models.OpenGaussCveList{CveNum: issueValue.CveNum, PackName: issueValue.RepoName, Status: 3} ogcErr := models.QueryReleaseCve(&ogc, "CveNum", "PackName", "Status") if ogc.Id > 0 { models.UpdateIssueStatus(issueValue, 2) @@ -506,27 +507,15 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) er if issueValue.OrganizationID == 3 { // Get branch information // Query the repo that needs to submit an issue - cveList := strings.Split(issueValue.CveVersion, ",") - if len(cveList) > 0 { - for _, cl := range cveList { - ms := models.MindSporeYaml{PackageName: issueValue.PackName, Version: cl} - sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") - if ms.Id > 0 { - path = ms.Repo - break - } - logs.Info("GetMindSporeYaml, sporeErr: ", sporeErr) - } - } taskhandler.CreateBrandAndTags(accessToken, owner, path, issueValue.OrganizationID) } // Query issue template var it models.IssueTemplate it.CveNum = issueValue.CveNum it.OwnedVersion = issueValue.CveVersion - it.OwnedComponent = issueValue.PackName + it.Repo = issueValue.PackName it.CveId = issueValue.CveId - templateErr := models.GetIssueTemplateByColName(&it, "CveNum", "OwnedVersion", "OwnedComponent", "CveId") + templateErr := models.GetIssueTemplateByColName(&it, "CveNum", "OwnedVersion", "Repo", "CveId") if templateErr == nil && it.TemplateId > 0 { it.NVDScore = sr.NVDScore it.NVDVector = sr.NvectorVule @@ -664,8 +653,12 @@ func GaussIssue(issueValue models.VulnCenter, accessToken, owner, path, assignee } branchList := make([]string, 0) errBrands := errors.New("") - gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") - path = gaussIssuePath + if len(issueValue.PackName) < 2 { + gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") + path = gaussIssuePath + } else { + path = issueValue.PackName + } gaussBranchPath := beego.AppConfig.String("opengauss::gauss_branch_path") // Get branch information branchList, errBrands = taskhandler.GetBranchesInfo(accessToken, owner, gaussBranchPath, issueValue.OrganizationID) @@ -716,19 +709,6 @@ func MindSporeIssue(issueValue models.VulnCenter, accessToken, owner, path, assi } else { logs.Error("GetSporeSecurityReviewerList, rlerr: ", rlerr) } - // Query the repo that needs to submit an issue - cveList := strings.Split(issueValue.CveVersion, ",") - if len(cveList) > 0 { - for _, cl := range cveList { - ms := models.MindSporeYaml{PackageName: issueValue.PackName, Version: cl} - sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") - if ms.Id > 0 { - path = ms.Repo - break - } - logs.Info("GetMindSporeYaml, sporeErr: ", sporeErr) - } - } branchList := make([]string, 0) // Get branch information branchList = taskhandler.CreateBrandAndTags(accessToken, owner, path, issueValue.OrganizationID) diff --git a/cve-vulner-manager/taskhandler/common.go b/cve-vulner-manager/taskhandler/common.go index 5320108b0f556eddc60b868c245555dd9dfe5265..b63f3313d6b6b3e40ab51b3846e71e099d9a1f6e 100644 --- a/cve-vulner-manager/taskhandler/common.go +++ b/cve-vulner-manager/taskhandler/common.go @@ -791,7 +791,7 @@ func CreateIssueBody(accessToken, owner, path, assignee string, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, oVector, affectedVersion, abiVersion) } else { - body = fmt.Sprintf(bodyUpTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + body = fmt.Sprintf(bodyUpTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, oVector, affectedVersion) } @@ -816,7 +816,7 @@ func CreateIssueBody(accessToken, owner, path, assignee string, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, affectedVersion, abiVersion) } else { - body = fmt.Sprintf(bodyTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + body = fmt.Sprintf(bodyTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, affectedVersion) } @@ -841,7 +841,7 @@ func CreateIssueBody(accessToken, owner, path, assignee string, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, affectedVersion, abiVersion) } else { - body = fmt.Sprintf(bodyTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + body = fmt.Sprintf(bodyTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, affectedVersion) } @@ -871,7 +871,7 @@ func CreateIssueBody(accessToken, owner, path, assignee string, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, oVector, affectedVersion, abiVersion) } else { - body = fmt.Sprintf(bodyUpTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + body = fmt.Sprintf(bodyUpTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, oVector, affectedVersion) } @@ -896,7 +896,7 @@ func CreateIssueBody(accessToken, owner, path, assignee string, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, affectedVersion, abiVersion) } else { - body = fmt.Sprintf(bodyTplx, cveNumber, cve.PackName, cve.CveVersion, scoreType, nveScore, nveVector, + body = fmt.Sprintf(bodyTplx, cveNumber, cve.RepoName, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, affectedVersion) } diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index 204162a38fdec6a14281597ea8c9df2d9acef544..eef118c3757e277ff660c7d4ae2f74b7191313c2 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -23,7 +23,7 @@ func CreateIssueData(issueTemp *models.IssueTemplate, cve models.VulnCenter, path, assignee, issueType, labels, owner string) { issueTemp.CveId = cve.CveId issueTemp.CveNum = cve.CveNum - issueTemp.OwnedComponent = cve.PackName + issueTemp.OwnedComponent = cve.RepoName issueTemp.OwnedVersion = cve.CveVersion if issueTemp.TemplateId == 0 { issueTemp.MtAuditFlag = 1 @@ -114,9 +114,9 @@ func OPenCheckWhetherIssue(cveNumber, repoPath, owner, accessToken string, organ } issueTmp := models.IssueTemplate{} issueTmp.CveNum = cveNumber - issueTmp.OwnedComponent = repoPath + issueTmp.Repo = repoPath issueTmp.CveId = vc.CveId - _ = models.GetIssueTemplateByColName(&issueTmp, "CveNum", "OwnedComponent", "CveId") + _ = models.GetIssueTemplateByColName(&issueTmp, "CveNum", "Repo", "CveId") if issueTmp.TemplateId > 0 && len(issueTmp.IssueNum) > 1 { issueErr, issueBody := GetGiteeIssue(accessToken, owner, repoPath, issueTmp.IssueNum) if issueErr == nil { @@ -408,23 +408,12 @@ func UpdateIssueToGit(accessToken string, owner string, path string, if cve.OrganizationID == 2 { owner = beego.AppConfig.String("opengauss::gauss_owner") accessToken = beego.AppConfig.String("opengauss::git_gauss_token") - path = beego.AppConfig.String("opengauss::gauss_issue_path") + if len(path) < 2 { + path = beego.AppConfig.String("opengauss::gauss_issue_path") + } } else if cve.OrganizationID == 3 { owner = beego.AppConfig.String("mindspore::mindspore_owner") accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") - // Query the repo that needs to submit an issue - cveList := strings.Split(cve.CveVersion, ",") - if len(cveList) > 0 { - for _, cl := range cveList { - ms := models.MindSporeYaml{PackageName: cve.PackName, Version: cl} - sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") - if ms.Id > 0 { - path = ms.Repo - break - } - logs.Info("GetMindSporeYaml, sporeErr: ", sporeErr) - } - } } assigneeGite := "" if its.IssueNum != "" && len(its.IssueNum) > 2 { @@ -794,7 +783,10 @@ func CreateSecNoticeData(sec *models.SecurityNotice, iss models.VulnCenter, sec.CveId = iss.CveId sec.CveNum = iss.CveNum opScoreLeve := models.OpenEulerScoreProc(opScore) - sec.Introduction = "An update for " + iss.PackName + " is now available for " + branchs + "." + if len(iss.RepoName) < 1 { + iss.RepoName = iss.PackName + } + sec.Introduction = "An update for " + iss.RepoName + " is now available for " + branchs + "." if iss.OrganizationID == 3 { sec.Theme = sec.Introduction[:len(sec.Introduction)-1] + ".\n\n" + "MindSpore Security has rated this" + " update as having a security impact of " + strings.ToLower(opScoreLeve) + ". A Common Vunlnerability" + diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index 0c429bd04f81871c7c8c5af9e42d07d067631403..59c245a5d965a6c3537ab6ae00370d400a82e30f 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -49,7 +49,8 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, if len(CveRes.RepairTime) < 2 && len(cveData.RepairTime) > 2 { CveRes.RepairTime = cveData.RepairTime } - CveRes.PackName = cveData.PackName + //CveRes.PackName = cveData.PackName + CveRes.RepoName = cveData.PackName CveRes.CveUrl = cveRef + cveData.CveNum CveRes.CveLevel = cveData.CveLevel CveRes.OrganizationID = organizationID @@ -63,8 +64,8 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, if (CveRes.DataSource == 4 || CveRes.DataSource == 3) && CveRes.Status != 0 && CveRes.Status != 1 { issueTmp := models.IssueTemplate{CveId: CveRes.CveId, - OwnedComponent: CveRes.PackName, OwnedVersion: CveRes.CveVersion} - err := models.GetIssueTemplateByColName(&issueTmp, "CveId", "OwnedComponent", "OwnedVersion") + Repo: CveRes.PackName, OwnedVersion: CveRes.CveVersion} + err := models.GetIssueTemplateByColName(&issueTmp, "CveId", "Repo", "OwnedVersion") if err != nil { if CveRes.DataSource == 3 { CveRes.Status = 0 @@ -245,7 +246,7 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, return true, nil } -func InsertCveExcelGroups(cveData models.OriginExcel, cveRef string, openeulerNum int, +func InsertCveExcelGroups(cveData models.OriginExcel, cveRef, repoName string, openeulerNum int, goe models.GitPackageInfo, organizationID int8) (bool, error) { var vul models.VulnCenter vul.CveNum = cveData.CveNum @@ -253,7 +254,8 @@ func InsertCveExcelGroups(cveData models.OriginExcel, cveRef string, openeulerNu vul.Status = 0 vul.CveVersion = cveData.CveVersion vul.RepairTime = cveData.RepairTime - vul.PackName = cveData.PackName + vul.PackName = repoName + vul.RepoName = cveData.PackName vul.CveUrl = cveRef + cveData.CveNum vul.CveLevel = cveData.CveLevel vul.DataSource = 3 @@ -356,7 +358,8 @@ func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum if len(CveRes.RepairTime) < 2 && len(cveData.PublishedDate) > 2 { CveRes.RepairTime = cveData.PublishedDate } - CveRes.PackName = pkList[0] + //CveRes.PackName = pkList[0] + CveRes.RepoName = pkList[0] CveRes.CveUrl = cveRef + cveData.CveNum CveRes.CveLevel = cveScV3.CveLevel CveRes.OrganizationID = organizationID @@ -599,7 +602,7 @@ func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum return true, nil } -func InsertCveGroups(cveData models.OriginUpstream, cveRef string, +func InsertCveGroups(cveData models.OriginUpstream, cveRef, repoNme string, openeulerNum int, cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, scopeType string, cveScV2 models.OriginUpstreamImpactScoreV2, @@ -612,7 +615,8 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef string, vul.Status = 0 vul.CveVersion = pkList[1] vul.RepairTime = cveData.PublishedDate - vul.PackName = pkList[0] + vul.RepoName = pkList[0] + vul.PackName = repoNme vul.CveUrl = cveRef + cveData.CveNum vul.CveLevel = cveScV3.CveLevel vul.DataSource = 1 @@ -834,8 +838,9 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) openGausVersion := make([]string, 0) mindSporeVersion := make([]string, 0) openEulerVersion := make([]string, 0) + versionList := make([]string, 0) if value != "" && len(value) > 0 { - versionList := strings.Split(value, ",") + versionList = strings.Split(value, ",") if len(versionList) > 0 { for _, ver := range versionList { if ver != "" && len(ver) > 1 { @@ -881,42 +886,113 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) organizationID = int8(3) organizationList = append(organizationList, organizationID) } + failFlag := false for _, orId := range organizationList { - CveRes, err := models.QueryCveByNum(cveData.CveNum, key, value, orId) - if err { - retVersion := AddCveVersion(orId, openGausVersion, - openEulerVersion, mindSporeVersion, CveRes.CveVersion) - pkList = append(pkList, retVersion) - lockx.Lock() - ok, err := UpdateCveGroups(cveData, cveRef, openeulernum, CveRes, cveDesc, cveScV3, goe, - scopeType, cveScV2, pkList, orId) - lockx.Unlock() - if !ok { - logs.Error("GenCveVuler, UpdateCveGroups, cveData: ", cveData, ", err: ", err) - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) - return false, errors.New("数据错误，暂时不处理") - } - } else { - retVersion := AddCveVersion(orId, openGausVersion, - openEulerVersion, mindSporeVersion, "") - pkList = append(pkList, retVersion) - lockx.Lock() - ok, err := InsertCveGroups(cveData, cveRef, openeulernum, cveDesc, cveScV3, goe, - scopeType, cveScV2, pkList, orId) - lockx.Unlock() - if !ok { - logs.Error("GenCveVuler, InsertCveGroups, cveData: ", cveData, ", err: ", err) - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) - return false, errors.New("数据错误，暂时不处理") - } + ok, addErr := AddOrSelectToCenter(key, cveRef, scopeType, value, cveData, + openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulernum, + cveDesc, cveScV3, goe, cveScV2, orId) + if !ok || addErr != nil { + logs.Error("AddOrSelectToCenter, addErr: ", addErr) + failFlag = true + continue } } + if failFlag { + logs.Error("Part of the data failed") + return false, errors.New("Part of the data failed") + } } } models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 2) return true, nil } +func AddOrSelectToCenter(packageName, cveRef, scopeType, value string, cveData models.OriginUpstream, + openGausVersion, openEulerVersion, mindSporeVersion, pkList []string, openeulerNum int, + cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, + cveScV2 models.OriginUpstreamImpactScoreV2, organizationID int8) (bool, error) { + if organizationID == 3 { + ms := models.MindSporeYaml{PackageName: packageName, Version: value} + msy, mErr := models.GetMindSporeYamlAll(&ms) + if len(msy) > 0 { + for _, my := range msy { + ok, dErr := AddOrDataToCenter(my.Repo, packageName, cveRef, scopeType, value, cveData, + openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) + return ok, dErr + } + } + } else { + logs.Error("MindSpore, mErr: ", mErr) + return false, mErr + } + } else if organizationID == 2 { + opy := models.OpenGussYaml{PackageName: packageName, Version: value} + ogy, oErr := models.GetOpengaussYamlAll(&opy) + if len(ogy) > 0 { + for _, gy := range ogy { + ok, dErr := AddOrDataToCenter(gy.Repo, packageName, cveRef, scopeType, value, cveData, + openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("openGauss, dErr: ", dErr) + return ok, dErr + } + } + } else { + logs.Error("openGauss, oErr: ", oErr) + return false, oErr + } + } else { + ok, dErr := AddOrDataToCenter(packageName, packageName, cveRef, scopeType, value, cveData, + openGausVersion, openEulerVersion, mindSporeVersion, pkList, openeulerNum, + cveDesc, cveScV3, goe, cveScV2, organizationID) + if !ok { + logs.Error("openEuler, dErr: ", dErr) + return ok, dErr + } + } + return true, nil +} + +func AddOrDataToCenter(repoNme, packageName, cveRef, scopeType, value string, cveData models.OriginUpstream, + openGausVersion, openEulerVersion, mindSporeVersion, pkList []string, openeulerNum int, + cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, + cveScV2 models.OriginUpstreamImpactScoreV2, organizationID int8) (bool, error) { + CveRes, err := models.QueryCveByNum(cveData.CveNum, repoNme, value, organizationID) + if err { + CveRes.RepoName = packageName + retVersion := AddCveVersion(organizationID, openGausVersion, + openEulerVersion, mindSporeVersion, CveRes.CveVersion) + pkList = append(pkList, retVersion) + lockx.Lock() + ok, err := UpdateCveGroups(cveData, cveRef, openeulerNum, CveRes, cveDesc, cveScV3, goe, + scopeType, cveScV2, pkList, organizationID) + lockx.Unlock() + if !ok { + logs.Error("GenCveVuler, UpdateCveGroups, cveData: ", cveData, ", err: ", err) + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") + } + } else { + retVersion := AddCveVersion(organizationID, openGausVersion, + openEulerVersion, mindSporeVersion, "") + pkList = append(pkList, retVersion) + lockx.Lock() + ok, err := InsertCveGroups(cveData, cveRef, repoNme, openeulerNum, cveDesc, cveScV3, goe, + scopeType, cveScV2, pkList, organizationID) + lockx.Unlock() + if !ok { + logs.Error("GenCveVuler, InsertCveGroups, cveData: ", cveData, ", err: ", err) + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") + } + } + return true, nil +} + func AddCveVersion(organizationID int8, openGausVersion, openEulerVersion, mindSporeVersion []string, cveVersion string) (retVersion string) { retVersion = cveVersion @@ -1008,36 +1084,93 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe logs.Error("SyncCveVuler, The repo exists, but the yaml data source does not exist,"+ " repo: ", cveData.PackName, cveData.CveVersion) } - + failFlag := false for _, orId := range organizationList { - CveRes, err := models.QueryCveByNum(cveData.CveNum, cveData.PackName, cveData.CveVersion, orId) - if err { - retVersion := AddCveVersion(orId, []string{cveData.CveVersion}, - []string{cveData.CveVersion}, []string{cveData.CveVersion}, CveRes.CveVersion) - cveData.CveVersion = retVersion - lockx.Lock() - ok, err := UpdateExcelCveGroups(cveData, cveRef, openeulerNum, CveRes, goe, orId) - lockx.Unlock() - if !ok { - logs.Error("SyncCveVuler, UpdateExcelCveGroups, cveData: ", cveData, ", err: ", err) - models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) - return false, errors.New("数据错误，暂时不处理") + exOk, exErr := AddExcelToCenter(cveData, cveRef, openeulerNum, goe, orId) + if !exOk || exErr != nil { + logs.Error("AddExcelToCenter, exErr: ", exErr) + failFlag = true + } + } + if failFlag { + return false, errors.New("数据错误，暂时不处理") + } + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 2) + return true, nil +} + +func AddExcelToCenter(cveData models.OriginExcel, cveRef string, openeulerNum int, + goe models.GitPackageInfo, organizationID int8) (bool, error) { + if organizationID == 3 { + ms := models.MindSporeYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + msy, mErr := models.GetMindSporeYamlAll(&ms) + if len(msy) > 0 { + for _, my := range msy { + ok, dErr := AddExDataToCenter(cveData, cveRef, my.Repo, openeulerNum, goe, organizationID) + if !ok { + logs.Error("MindSpore, dErr: ", dErr) + return ok, dErr + } } } else { - retVersion := AddCveVersion(orId, []string{cveData.CveVersion}, - []string{cveData.CveVersion}, []string{cveData.CveVersion}, "") - cveData.CveVersion = retVersion - lockx.Lock() - ok, err := InsertCveExcelGroups(cveData, cveRef, openeulerNum, goe, orId) - lockx.Unlock() - if !ok { - logs.Error("SyncCveVuler, InsertCveExcelGroups, cveData: ", cveData, ", err: ", err) - models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) - return false, errors.New("数据错误，暂时不处理") + logs.Error("MindSpore, mErr: ", mErr) + return false, mErr + } + } else if organizationID == 2 { + opy := models.OpenGussYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + ogy, oErr := models.GetOpengaussYamlAll(&opy) + if len(ogy) > 0 { + for _, gy := range ogy { + ok, dErr := AddExDataToCenter(cveData, cveRef, gy.Repo, openeulerNum, goe, organizationID) + if !ok { + logs.Error("openGauss, dErr: ", dErr) + return ok, dErr + } } + } else { + logs.Error("openGauss, oErr: ", oErr) + return false, oErr + } + } else { + ok, dErr := AddExDataToCenter(cveData, cveRef, cveData.PackName, openeulerNum, goe, organizationID) + if !ok { + logs.Error("openEuler, dErr: ", dErr) + return ok, dErr + } + } + return true, nil +} + +func AddExDataToCenter(cveData models.OriginExcel, + cveRef, repoName string, openeulerNum int, + goe models.GitPackageInfo, organizationID int8) (bool, error) { + CveRes, err := models.QueryCveByNum(cveData.CveNum, repoName, cveData.CveVersion, organizationID) + if err { + CveRes.RepoName = cveData.PackName + retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, + []string{cveData.CveVersion}, []string{cveData.CveVersion}, CveRes.CveVersion) + cveData.CveVersion = retVersion + lockx.Lock() + ok, err := UpdateExcelCveGroups(cveData, cveRef, openeulerNum, CveRes, goe, organizationID) + lockx.Unlock() + if !ok { + logs.Error("AddExDataToCenter, UpdateExcelCveGroups, cveData: ", cveData, ", err: ", err) + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") + } + } else { + retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, + []string{cveData.CveVersion}, []string{cveData.CveVersion}, "") + cveData.CveVersion = retVersion + lockx.Lock() + ok, err := InsertCveExcelGroups(cveData, cveRef, repoName, openeulerNum, goe, organizationID) + lockx.Unlock() + if !ok { + logs.Error("AddExDataToCenter, InsertCveExcelGroups, cveData: ", cveData, ", err: ", err) + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) + return false, errors.New("数据错误，暂时不处理") } } - models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 2) return true, nil } @@ -1148,6 +1281,7 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c vul.CveVersion = retVersion vul.RepairTime = "" vul.PackName = cveData.RepoPath + vul.RepoName = lop.Components vul.CveUrl = cveRef + cveData.CveNumber vul.IsExport = 0 vul.DataSource = 4 @@ -1289,7 +1423,7 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c issueTemp.CveId = cveid issueTemp.CveNum = vul.CveNum issueTemp.CveNum = cveData.CveNumber - issueTemp.OwnedComponent = cveData.RepoPath + issueTemp.OwnedComponent = lop.Components issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) issueTemp.NVDScore = nVDScore issueTemp.OpenEulerScore = openEulerScore @@ -1382,6 +1516,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c if vul.CveUrl == "" || len(vul.CveUrl) < 1 { vul.CveUrl = cveRef + cveData.CveNumber } + vul.RepoName = lop.Components vul.OrganizationID = cveData.OrganizationID owner = beego.AppConfig.String("gitee::owner") accessToken := beego.AppConfig.String("gitee::git_token") @@ -1655,7 +1790,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c if templateErr == nil && issueTemp.TemplateId > 0 { //issueTemp.CveNum = cveData.CveNumber if len(issueTemp.OwnedComponent) < 2 { - issueTemp.OwnedComponent = cveData.RepoPath + issueTemp.OwnedComponent = lop.Components } //issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) @@ -1687,6 +1822,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c } issueTemp.Solution = lop.AvoidScheme statusName := issueTemp.StatusName + issueTemp.Repo = cveData.RepoPath if issueTemp.IssueId != cveData.IssueId || issueTemp.IssueNum != cveData.Number { tmpIssueErr, tmpIssueBody := GetGiteeIssue(accessToken, owner, vul.PackName, issueTemp.IssueNum) cveIssueErr, cveIssueBody := GetGiteeIssue(accessToken, owner, vul.PackName, cveData.Number) @@ -1737,7 +1873,6 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c issueTemp.IssueStatus = 6 } issueTemp.Owner = owner - issueTemp.Repo = cveData.RepoPath if cveData.Title != "" { issueTemp.Title = cveData.Title } @@ -1745,7 +1880,9 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c issueTemp.CveLevel = vul.CveLevel } else { issueTemp.CveNum = cveData.CveNumber - issueTemp.OwnedComponent = cveData.RepoPath + issueTemp.OwnedComponent = lop.Components + issueTemp.Repo = cveData.RepoPath + issueTemp.Owner = owner issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) issueTemp.NVDScore = nVDScore openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) @@ -1800,8 +1937,6 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c issueTemp.Status = 4 issueTemp.IssueStatus = 6 } - issueTemp.Owner = owner - issueTemp.Repo = cveData.RepoPath if cveData.Title != "" { issueTemp.Title = cveData.Title } @@ -1900,11 +2035,21 @@ func GenCveVulerByIssue(cveData models.GiteOriginIssue, cveRef string, openeuler models.UpdateCveIssueStatusById(3, cveData.Id) return false, err } - goe, ok := models.QueryCveOpeneulerDetaildataByName(cveData.RepoPath, hole.Version) - if !ok || goe.DetailId == 0 { - logs.Error("Failed to get data, ", "Components: ", hole.Components, ",Version: ", hole.Version) + goe := models.GitPackageInfo{} + if organizationID == 1 { + goe, _ = models.QueryCveOpeneulerDetaildataByName(cveData.RepoPath, hole.Version) + if goe.DetailId == 0 { + logs.Error("Failed to get data, ", "Components: ", hole.Components, ",Version: ", hole.Version) + } + } else { + goe, _ = models.QueryCveOpeneulerDetaildataByName(hole.Components, hole.Version) + if goe.DetailId == 0 { + logs.Error("Failed to get data, ", "Components: ", hole.Components, ",Version: ", hole.Version) + } + } + if len(goe.Version) > 0 { + hole.Version = goe.Version } - hole.Version = goe.Version if cveData.State != "" && (strings.ToLower(cveData.State) == "rejected" || cveData.State == "已拒绝") { logs.Info("Process data whose issue status is rejected：", cveData) cvd, ok := models.QueryCveByNum(cveData.CveNumber, cveData.RepoPath, hole.Version, organizationID) diff --git a/cve-vulner-manager/taskhandler/gauss.go b/cve-vulner-manager/taskhandler/gauss.go index 8a90fe7b3430df901b1b8aa263be107fa90572c8..07e6cdb552fa3354473e57c6366be0102e8bfe2a 100644 --- a/cve-vulner-manager/taskhandler/gauss.go +++ b/cve-vulner-manager/taskhandler/gauss.go @@ -61,7 +61,7 @@ func GaussUnaffectIssue(branchVersion []string, gi GitInfo) { logs.Info("Continue to execute data, ogcErr: ", ogcErr) } // Get specific business data - el, gsErr := models.QueryGaussExportSaData(v.CveNum, v.IssueNum, v.OwnedComponent, v.IssueId) + el, gsErr := models.QueryGaussExportSaData(v.CveNum, v.IssueNum, v.Repo, v.IssueId) if gsErr != nil { logs.Error("QueryGaussExportSaData, err: ", gsErr, "tpl.CveNum, tpl.IssueNum: ", v.CveNum, v.IssueNum, v.OwnedComponent) @@ -182,6 +182,8 @@ func CreateReleaseCve(ogc *models.OpenGaussCveList, xmlp *models.ExcelExport, fl updateSlice = append(updateSlice, "Description") ogc.PackName = xmlp.OwnedComponent updateSlice = append(updateSlice, "PackName") + ogc.RepoName = xmlp.Repo + updateSlice = append(updateSlice, "RepoName") ogc.NVDScore = xmlp.NVDScore updateSlice = append(updateSlice, "NVDScore") ogc.OpenEulerScore = xmlp.OpenEulerScore @@ -274,7 +276,7 @@ func CreateReleaseCve(ogc *models.OpenGaussCveList, xmlp *models.ExcelExport, fl } func GaussFindUnaffVersion(xmlp *models.ExcelExport, versions string, gi GitInfo) bool { - branchArry, _ := GetBranchesInfo(gi.Token, gi.Owner, xmlp.OwnedComponent, 2) + branchArry, _ := GetBranchesInfo(gi.Token, gi.Owner, xmlp.Repo, 2) if len(branchArry) == 0 { gaussVersion := beego.AppConfig.String("opengauss::gauss_version") if len(gaussVersion) > 1 { @@ -386,14 +388,14 @@ func GaussHandleGiteData(c <-chan []IssueAndPkg, fExcel models.OpenGaussDownload func GaussAffectIssue(v IssueAndPkg, pkgList []string, fExcel models.OpenGaussDownloadFile, cveComponent *CveComponent) { for _, iv := range v.IssueMap { - tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo, IssueId: v.IssueId} - err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo", "issue_id") + tpl := models.IssueTemplate{IssueNum: iv.Number, IssueId: v.IssueId} + err := models.GetIssueTemplateByColName(&tpl, "issue_num", "issue_id") if err != nil { - logs.Error("GetIssueTemplateByColName, ----", err, iv.Number, iv.Repo, v.IssueId) + logs.Error("GetIssueTemplateByColName, ----", err, iv.Number, v.Repo, v.IssueId) continue } //save data to db - el, err := models.QueryGaussExportSaData(tpl.CveNum, tpl.IssueNum, tpl.OwnedComponent, tpl.IssueId) + el, err := models.QueryGaussExportSaData(tpl.CveNum, tpl.IssueNum, tpl.Repo, tpl.IssueId) if err != nil { logs.Error("QueryGaussExportSaData, err: ", err, "tpl.CveNum, tpl.IssueNum: ", tpl.CveNum, tpl.IssueNum, tpl.OwnedComponent) @@ -741,7 +743,7 @@ func GuassGetDateByGite(pkgList []models.GaussExcelTag, c chan<- []IssueAndPkg, defer wgTrigger.Done() token := beego.AppConfig.String("opengauss::git_gauss_token") owner := beego.AppConfig.String("opengauss::gauss_owner") - gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") + //gaussIssuePath := beego.AppConfig.String("opengauss::gauss_issue_path") st := util.TimeStrToInt(startTime, "2006-01-02") chData := make([]IssueAndPkg, 0) for _, v := range pkgList { @@ -755,7 +757,7 @@ func GuassGetDateByGite(pkgList []models.GaussExcelTag, c chan<- []IssueAndPkg, for _, isTemp := range issueTemp { //logs.Info("isTemp===>", isTemp) var prList []models.PullRequestIssue - prList = getGaussRepoIssueAllPR(affectBranch, token, owner, gaussIssuePath, st, rt, isTemp) + prList = getGaussRepoIssueAllPR(affectBranch, token, owner, isTemp.Repo, st, rt, isTemp) //prList := getGaussRepoIssueAllPR(affectBranch, token, owner, gaussIssuePath, st, rt, isTemp) //get pull request related issue repoIssue := make(map[int64]models.PullRequestIssue, 0) @@ -829,7 +831,7 @@ func getGaussRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, pr.Id = int64(v["id"].(float64)) pr.Number = isTemp.IssueNum pr.CveNumber = isTemp.CveNum - pr.Repo = repo + pr.Repo = v["base"].(map[string]interface{})["repo"].(map[string]interface{})["path"].(string) pr.Branch = affectBranch pr.BrFlag = true prList = append(prList, pr) diff --git a/cve-vulner-manager/taskhandler/hook.go b/cve-vulner-manager/taskhandler/hook.go index d6bc34e9127aa99b91825a9098c1e0e3e6755231..e9e8a06fd872bc7c81f1c6c5a660853afbe5792d 100644 --- a/cve-vulner-manager/taskhandler/hook.go +++ b/cve-vulner-manager/taskhandler/hook.go @@ -246,6 +246,20 @@ func CheckAffectVerComplete(affectedVersion, packageName, version string, organi } else { return unFixValue } + if organizateId == 3 { + tmpList := make([]string, 0) + for _, brand := range affectBranchsxList { + mdbt := models.MindSporeBrandTags{PackageName: packageName, Brand: brand} + mbList := models.QueryMindSporeTagsByBrand(&mdbt) + if len(mbList) > 0 { + for _, mb := range mbList { + tmpList = append(tmpList, mb.Tags) + } + } + } + affectBranchsxList = affectBranchsxList[:0] + affectBranchsxList = append(affectBranchsxList, tmpList...) + } if affectedVersion != "" && len(affectedVersion) > 1 { affectProductList = strings.Split(affectedVersion, ",") } diff --git a/cve-vulner-manager/taskhandler/issue.go b/cve-vulner-manager/taskhandler/issue.go index 4b6e9abb47fd7504d2fa36ce6629181903d6f39a..83c44512d78149424d754229e92cf94eca94568a 100644 --- a/cve-vulner-manager/taskhandler/issue.go +++ b/cve-vulner-manager/taskhandler/issue.go @@ -22,8 +22,8 @@ func ProcIssueIsExists(prnum int) error { } for _, v := range vc { cveId = v.CveId - issueTmp := models.IssueTemplate{CveId: v.CveId, OwnedComponent: v.PackName, CveNum: v.CveNum} - tempErr := models.GetIssueTemplateByColName(&issueTmp, "CveId", "OwnedComponent", "CveNum") + issueTmp := models.IssueTemplate{CveId: v.CveId, Repo: v.PackName, CveNum: v.CveNum} + tempErr := models.GetIssueTemplateByColName(&issueTmp, "CveId", "Repo", "CveNum") if tempErr != nil { continue } @@ -32,25 +32,15 @@ func ProcIssueIsExists(prnum int) error { issueData := map[string]interface{}{} owner := beego.AppConfig.String("gitee::owner") accessToken := beego.AppConfig.String("gitee::git_token") - path := issueTmp.OwnedComponent + path := issueTmp.Repo if v.OrganizationID == 3 { owner = beego.AppConfig.String("mindspore::mindspore_owner") accessToken = beego.AppConfig.String("mindspore::git_mindspore_token") // Query the repo that needs to submit an issue - cveList := strings.Split(v.CveVersion, ",") - if len(cveList) > 0 { - for _, cl := range cveList { - ms := models.MindSporeYaml{PackageName: v.PackName, Version: cl} - sporeErr := models.GetMindSporeYaml(&ms, "PackageName", "Version") - if ms.Id > 0 { - path = ms.Repo - break - } - logs.Info("GetMindSporeYaml, sporeErr: ", sporeErr) - } - } } else if v.OrganizationID == 2 { - path = beego.AppConfig.String("opengauss::gauss_issue_path") + if len(path) < 2 { + path = beego.AppConfig.String("opengauss::gauss_issue_path") + } owner = beego.AppConfig.String("opengauss::gauss_owner") accessToken = beego.AppConfig.String("opengauss::git_gauss_token") } diff --git a/cve-vulner-manager/taskhandler/issuestatistics.go b/cve-vulner-manager/taskhandler/issuestatistics.go index 2908b279847d7af8f856d654336470130ac79ab3..dd28fe4555837b82084d9d3324351cd7f93f48c9 100644 --- a/cve-vulner-manager/taskhandler/issuestatistics.go +++ b/cve-vulner-manager/taskhandler/issuestatistics.go @@ -313,7 +313,7 @@ func ProcSecLinkTemplate(beforeDate, prcnum int, owner, accessToken string) erro templateId = temp.TemplateId cveCenter := models.VulnCenter{CveId: temp.CveId, CveNum: temp.CveNum} cveErr := models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") - if cveErr != nil { + if cveErr != nil || cveCenter.OrganizationID != 1 { continue } // Determine whether cve has been processed diff --git a/cve-vulner-manager/taskhandler/oricvecheck.go b/cve-vulner-manager/taskhandler/oricvecheck.go index 0e61edddb0233aa4ac0e620fc22d5e2fd1c827eb..f9e6d7d06453c5873cf00dfd5fc443a7301cad81 100644 --- a/cve-vulner-manager/taskhandler/oricvecheck.go +++ b/cve-vulner-manager/taskhandler/oricvecheck.go @@ -83,8 +83,8 @@ func UpdateAbnCveStatus(prcNum, days int, cveSt string) (string, error) { if err == nil && len(cveData) > 0 { for _, cd := range cveData { cveId = cd.CveId - issueTmp := models.IssueTemplate{CveNum: cd.CveNum, CveId: cd.CveId, OwnedComponent: cd.PackName} - err := models.GetIssueTemplateByColName(&issueTmp, "CveNum", "CveId", "OwnedComponent") + issueTmp := models.IssueTemplate{CveNum: cd.CveNum, CveId: cd.CveId, Repo: cd.PackName} + err := models.GetIssueTemplateByColName(&issueTmp, "CveNum", "CveId", "Repo") if issueTmp.TemplateId < 1 { cd.Status = 0 update := models.UpdateVulnCenter(&cd, "Status")