From 5f9a314a40c45e74c90a457c34a8c2517294f056 Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Thu, 26 Dec 2024 18:59:23 +0800 Subject: [PATCH 1/7] reopen anomalous issues and handle normal issues --- cve-vulner-manager/conf/app.conf | 3 + cve-vulner-manager/conf/product_app.conf | 3 + cve-vulner-manager/models/issue.go | 17 +- cve-vulner-manager/task/inittask.go | 23 +- cve-vulner-manager/task/issue.go | 479 ++++++++++++++++++++++- 5 files changed, 514 insertions(+), 11 deletions(-) diff --git a/cve-vulner-manager/conf/app.conf b/cve-vulner-manager/conf/app.conf index 2728bd2..119d113 100644 --- a/cve-vulner-manager/conf/app.conf +++ b/cve-vulner-manager/conf/app.conf @@ -131,6 +131,9 @@ setissueprocdate = */10 * * * * * releaseUnaffectedCveflag = 2 releaseUnaffectedCve = 0 0 11 * * 1 +dealAnomalousflag = 1 +dealAnomalousIssue = 0 2 * * 5 * + [gitee] #owner = cve-test #owner = src-openeuler diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index f5e401e..9e67050 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -134,6 +134,9 @@ syncissuedate = 0 0 7,13 * * * releaseUnaffectedCveflag = 1 releaseUnaffectedCve = 0 0 11 * * * +dealAnomalousflag = 1 +dealAnomalousIssue = 0 2 * * 5 * + [gitee] owner = src-openeuler diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go index c9f0a1a..382d331 100644 --- a/cve-vulner-manager/models/issue.go +++ b/cve-vulner-manager/models/issue.go @@ -13,6 +13,12 @@ import ( "github.com/astaxie/beego/orm" ) +const ( + statusCompleted = 3 + month = 6 + owner = "src-openeuler" +) + var mutex sync.Mutex // QueryIssueCveByNum query issue by cve num @@ -786,6 +792,13 @@ func (t *IssueTemplate) IsIssueWithAnalysisVersion() bool { // IsIssueComplete returns whether the issue is completed. func (t *IssueTemplate) IsIssueComplete() bool { - const StatusCompleted = 3 - return t.Status == StatusCompleted + return t.Status == statusCompleted +} + +func ListHalfYearCompleteIssues() (issueTemps []IssueTemplate, err error) { + sql := `select * from cve_issue_template where status = %d and create_time > DATE_SUB(NOW(), INTERVAL %d MONTH) and owner = %s` + sql = fmt.Sprintf(sql, statusCompleted, month, owner) + o := orm.NewOrm() + _, err = o.Raw(sql).QueryRows(&issueTemps) + return } diff --git a/cve-vulner-manager/task/inittask.go b/cve-vulner-manager/task/inittask.go index f2c06ff..a611a94 100644 --- a/cve-vulner-manager/task/inittask.go +++ b/cve-vulner-manager/task/inittask.go @@ -6,7 +6,7 @@ import ( "github.com/astaxie/beego/toolbox" ) -//CheckOriCveTask Verify the original cve data +// CheckOriCveTask Verify the original cve data func CheckOriCveTask(oriCveCheck string) { logs.Info("The task of verifying the original cve data starts...") CheckTask := toolbox.NewTask("CheckOriCve", oriCveCheck, CheckOriCve) @@ -14,7 +14,7 @@ func CheckOriCveTask(oriCveCheck string) { logs.Info("End of verifying original cve data task...") } -//InitYamlTask Get yaml data source +// InitYamlTask Get yaml data source func InitYamlTask(getYaml string) { logs.Info("Get the yaml data source task started...") yamlTask := toolbox.NewTask("GetYamlData", getYaml, GetYamlData) @@ -22,7 +22,7 @@ func InitYamlTask(getYaml string) { logs.Info("End of the task of obtaining yaml data source...") } -//InitEulerYamlTask Get yaml data source +// InitEulerYamlTask Get yaml data source func InitEulerYamlTask(eulergetymal string) { logs.Info("Get the euleryaml data source task started...") eulerYamlTask := toolbox.NewTask("GetEulerYamlData", eulergetymal, GetEulerYamlData) @@ -78,7 +78,7 @@ func CreateHookTask(createHook string) { logs.Info("End of execution to create all webhook tasks...") } -//GenSAExcelTask Execute issue to generate excel task start +// GenSAExcelTask Execute issue to generate excel task start func GenSAExcelTask(genExcel string) { logs.Info("Execute issue to generate excel task start...") genExcelTask := toolbox.NewTask("GenExcelData", genExcel, GenExcelData) @@ -150,7 +150,7 @@ func PrintLogTask(printLog string) { logs.Info("Execute log task task end...") } -//Statistics of issues created in different communities, uncreated issues are created +// Statistics of issues created in different communities, uncreated issues are created func IssueCommunityStatistTask(issuecommunity string) { logs.Info("Community issue statistics task started...") issueStatTask := toolbox.NewTask("CommunityIssueStatist", issuecommunity, CommunityIssueStatist) @@ -158,7 +158,7 @@ func IssueCommunityStatistTask(issuecommunity string) { logs.Info("Community issue statistics task is over...") } -//Check whether the issue label is reasonable +// Check whether the issue label is reasonable func IssueLabelCheckTask(issuelabelcheck string) { issueLaCheckTask := toolbox.NewTask("IssueLabelCheck", issuelabelcheck, IssueLabelCheck) toolbox.AddTask("IssueLabelCheck", issueLaCheckTask) @@ -183,6 +183,10 @@ func ReleaseUnaffetcdCveTask(spec string) { toolbox.AddTask("ReleaseUnaffetcdCveTask", toolbox.NewTask("ReleaseUnaffetcdCveTask", spec, ReleaseUnaffectedCve)) } +func DealAnomalousIssueTask(dealanomalousissue string) { + toolbox.AddTask("DealAnomalousIssueTask", toolbox.NewTask("DealAnomalousIssueTask", dealanomalousissue, DealAnomalousIssues)) +} + // start task func StartTask() { toolbox.StartTask() @@ -192,7 +196,7 @@ func StopTask() { toolbox.StopTask() } -//InitTask Timing task initialization +// InitTask Timing task initialization func InitTask() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -347,5 +351,10 @@ func InitTask() bool { ReleaseUnaffetcdCveTask(BConfig.String("crontab::releaseUnaffectedCve")) } + dealAnomalousFlag, sErr := BConfig.Int("crontab::dealAnomalousflag") + if dealAnomalousFlag == 1 && sErr == nil { + DealAnomalousIssueTask(BConfig.String("crontab::dealAnomalousIssue")) + } + return true } diff --git a/cve-vulner-manager/task/issue.go b/cve-vulner-manager/task/issue.go index bdf3a03..4b3aee6 100644 --- a/cve-vulner-manager/task/issue.go +++ b/cve-vulner-manager/task/issue.go @@ -3,19 +3,62 @@ package task import ( "bytes" "encoding/json" + "encoding/xml" + "errors" + "fmt" "net/http" "strconv" "strings" "time" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + sdk "github.com/opensourceways/go-gitee/gitee" "github.com/opensourceways/server-common-lib/utils" + "k8s.io/apimachinery/pkg/util/sets" "cvevulner/common" + "cvevulner/cve-ddd/infrastructure/bulletinimpl" + "cvevulner/cve-ddd/infrastructure/obsimpl" "cvevulner/models" "cvevulner/taskhandler" + "cvevulner/util" +) - "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" +const ( + referenceUrl = "https://gitee.com/help/articles/4142" + endpoint = "https://api-cve.openeuler.org" + defaultClientTimeout = 3 + modifiedCvrfTxt = "modified-cvrf.txt" + cvssV3 = "v3" + statusOpen = "open" +) + +var ( + mapVector = map[string]string{ + "AV:N": "Network", + "AV:A": "Adjacent Network", + "AV:L": "Local", + "AV:P": "Physical", + "AC:L": "Low", + "AC:H": "High", + "PR:N": "None", + "PR:L": "Low", + "PR:H": "High", + "UI:N": "None", + "UI:R": "Required", + "S:U": "Unchanged", + "S:C": "Changed", + "C:N": "None", + "C:L": "Low", + "C:H": "High", + "I:N": "None", + "I:L": "Low", + "I:H": "High", + "A:N": "None", + "A:L": "Low", + "A:H": "High", + } ) // Verify whether the issue on gitee has been deleted @@ -411,3 +454,435 @@ func getPlanData() []PlanDataOfMaJun { return data } + +func DealAnomalousIssues() error { + defer common.Catchs() + + issueTemps, issueErr := models.ListHalfYearCompleteIssues() + if len(issueTemps) == 0 { + logs.Error("list half year created and status is complete issues occurred err : ", issueErr) + return issueErr + } + + _, token := common.GetOwnerAndToken("", 1) + + normalIssues, anomalousIssues, anomalousIssueTips := processIssues(issueTemps, token) + + handleAnomalousIssues(anomalousIssues, anomalousIssueTips, token) + + handleNormalIssues(normalIssues) + + return nil +} + +// 筛选出不满足关闭条件的issue，这种issue需要重新打开并评论提示 +// 满足关闭条件但issue实际没有关闭的情况基本没有，所以不考虑处理这种issue +func processIssues(issueTemps []models.IssueTemplate, token string) ( + []models.IssueTemplate, + []models.IssueTemplate, + map[int64]string, +) { + normalIssues := make([]models.IssueTemplate, 0) + anomalousIssues := make([]models.IssueTemplate, 0) + anomalousIssueTips := make(map[int64]string) + + for _, issueTemp := range issueTemps { + if issueTemp.OpenEulerScore <= 0 { + tip := "openeuler评分不能为0！" + anomalousIssues = append(anomalousIssues, issueTemp) + anomalousIssueTips[issueTemp.TemplateId] = tip + continue + } + + score, err := models.QueryIssueScore(issueTemp.CveId) + if err != nil { + logs.Error("query issue score error: ", err.Error()) + continue + } + + // 如果score type 为 v3, 则检查 cvss v3 vector 和 score 是否一致 + if score.ScoreType == cvssV3 { + if util.CalculateCVSSV3BaseScore(issueTemp.OpenEulerVector) != issueTemp.OpenEulerScore { + tip := fmt.Sprintf("@%v cvss v3 vector: %v and score: %f are not consistent", + issueTemp.Assignee, issueTemp.OpenEulerVector, issueTemp.OpenEulerScore) + + anomalousIssues = append(anomalousIssues, issueTemp) + anomalousIssueTips[issueTemp.TemplateId] = tip + continue + } + } + + cveCenter := models.VulnCenter{CveId: issueTemp.CveId, CveNum: issueTemp.CveNum} + err = models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") + if err != nil { + logs.Error("get vuln center error: ", err.Error()) + continue + } + + // 检查issue的影响性分析是否完成 + if msg, _, ok := taskhandler.CheckIssueAnalysisComplete(&issueTemp, cveCenter.OrganizationID); !ok { + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + tip := fmt.Sprintf("@%v %v", issueTemp.Assignee, msg) + na + + anomalousIssues = append(anomalousIssues, issueTemp) + anomalousIssueTips[issueTemp.TemplateId] = tip + continue + } + + // 检查issue的受影响分支关联的PR是否合入 + versions := strings.Split(issueTemp.AffectedVersion, ",") + affectedVersions := make([]string, 0) + for _, v := range versions { + if !strings.Contains(v, "不受影响") { + affectedVersions = append(affectedVersions, v) + } + } + + relatedPRNotMerged, err := checkRelatedPR(issueTemp, token, affectedVersions) + if err != nil { + logs.Error("check related PR error: ", err.Error()) + continue + } + + if len(relatedPRNotMerged) > 0 { + tip := fmt.Sprintf("@%s\n"+ + "关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #%s\n"+ + "受影响分支: %s\n"+ + "具体操作参考: %s\n", + issueTemp.Assignee, + issueTemp.IssueNum, + strings.Join(relatedPRNotMerged, "/"), + referenceUrl) + + anomalousIssues = append(anomalousIssues, issueTemp) + anomalousIssueTips[issueTemp.TemplateId] = tip + continue + } + + normalIssues = append(normalIssues, issueTemp) + + // 防止调用giteeAPI太频繁，出现429错误 + time.Sleep(300 * time.Millisecond) + } + + return normalIssues, anomalousIssues, anomalousIssueTips +} + +// 与gitee平台进行交互，重新打开异常关闭的issue并评论提示 +func handleAnomalousIssues(anomalousIssues []models.IssueTemplate, anomalousIssueTips map[int64]string, token string) { + for i := range anomalousIssues { + issueNum := anomalousIssues[i].IssueNum + owner := anomalousIssues[i].Owner + repo := anomalousIssues[i].Repo + cveNum := anomalousIssues[i].CveNum + + anomalousIssues[i].Status = 1 + anomalousIssues[i].StatusName = statusOpen + issueOption := taskhandler.IssueOptions{Token: token, Repo: repo, State: anomalousIssues[i].StatusName} + requestBody, _ := json.Marshal(issueOption) + url := "https://gitee.com/api/v5/repos/" + owner + "/issues/" + issueNum + + // 重新打开issue + _, err := util.HTTPPatch(url, string(requestBody)) + if err != nil { + logs.Error("UpdateIssueToGit, Update issue failed, cveNum: ", cveNum, "err: ", err) + continue + } + + // 更新数据库表CveIssueTemplate + err = models.UpdateIssueTemplate(&anomalousIssues[i], "Status", "StatusName") + if err != nil { + logs.Error("UpdateIssueTemplate, Update issue failed, cveNum: ", cveNum, "err: ", err) + continue + } + + // 评论提示相关错误信息 + tip := anomalousIssueTips[anomalousIssues[i].TemplateId] + taskhandler.AddCommentToIssue(tip, issueNum, owner, repo, token) + + // 防止调用giteeAPI太频繁，出现429错误 + if i < len(anomalousIssues)-1 { + time.Sleep(300 * time.Millisecond) + } + } +} + +// 正常关闭的issue需要校验openeuler评分是否和cve官网一致，如果不一致，需要更新cve官网评分和cvrf文件内容 +func handleNormalIssues(normalIssues []models.IssueTemplate) { + obs := obsimpl.Instance() + cvrfdir := beego.AppConfig.String("obs::download_cvrf_dir") + modifiedCvrfFiles := make([]string, 0) + + for _, issue := range normalIssues { + cve, err := getCveByIdAndPackageName(issue.CveNum, issue.Repo) + if err != nil { + logs.Error("getCveByIdAndPackageName error: ", err.Error(), + "cveNum: ", issue.CveNum, "repo: ", issue.Repo) + continue + } + + eulerScore := strconv.FormatFloat(issue.OpenEulerScore, 'f', -1, 64) + + if cve.CvsssCoreoe == eulerScore { + continue + } + + cve.CvsssCoreoe = eulerScore + + // 检查len(vectors) == 8 + vectors := strings.Split(issue.OpenEulerVector, "/") + if len(vectors) != 8 { + continue + } + + cve.AttackVectoroe = mapVector[vectors[0]] + cve.AttackComplexityoe = mapVector[vectors[1]] + cve.PrivilegesRequiredoe = mapVector[vectors[2]] + cve.UserInteractionoe = mapVector[vectors[3]] + cve.Scopeoe = mapVector[vectors[4]] + cve.Confidentialityoe = mapVector[vectors[5]] + cve.Integrityoe = mapVector[vectors[6]] + cve.Availabilityoe = mapVector[vectors[7]] + + // 更新cvedatabase + err = updateCveDatabase(cve.CveDatabase) + if err != nil { + logs.Error("updateCveDatabase error: ", err.Error(), "cveDatabase: ", cve.CveDatabase.CveId) + continue + } + + // 修改cvrf文件 + if cve.SecurityNoticeNo == "" { + continue + } + + securityNoticeNo := strings.Split(cve.SecurityNoticeNo, "-") + if len(securityNoticeNo) != 4 { + continue + } + + year := securityNoticeNo[2] + + cvrfFilePath := fmt.Sprintf("%s%s/%s", cvrfdir, year, cve.SecurityNoticeNo) + + content, err := obs.Download(cvrfFilePath) + if err != nil { + logs.Error("download cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfFilePath) + continue + } + + var cvrf bulletinimpl.Cvrf + err = xml.Unmarshal(content, &cvrf) + if err != nil { + logs.Error("unmarshal cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfFilePath) + continue + } + + for i := range cvrf.Vulnerability { + if cvrf.Vulnerability[i].CVE == issue.CveNum { + cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.BaseScore = eulerScore + cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.Vector = issue.OpenEulerVector + + break + } + } + + uploadBys, err := xml.MarshalIndent(cvrf, "", " ") + if err != nil { + logs.Error("marshal cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfFilePath) + continue + } + + headerBytes := []byte(xml.Header) + headerBytes = append(headerBytes, uploadBys...) + + err = obs.Upload(cvrfFilePath, headerBytes) + if err != nil { + logs.Error("upload cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfFilePath) + continue + } + + modifiedCvrfFiles = append(modifiedCvrfFiles, cvrfFilePath) + } + + if len(modifiedCvrfFiles) > 0 { + content := strings.Join(modifiedCvrfFiles, "\n") + + err := obs.Upload(cvrfdir+modifiedCvrfTxt, []byte(content)) + if err != nil { + logs.Error("upload cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfdir+modifiedCvrfTxt) + } + } +} + +// 检查issue受影响的分支是否都已经关联PR并且PR都合并 +func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []string) (relatedPRNotMerged []string, err error) { + endpoint := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests?access_token=%s&repo=%s", + issueTemp.Owner, issueTemp.IssueNum, token, issueTemp.Repo, + ) + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return relatedPRNotMerged, err + } + + var prs []sdk.PullRequest + cli := utils.NewHttpClient(3) + bytes, _, err := cli.Download(req) + if err != nil { + return relatedPRNotMerged, err + } + + if err := json.Unmarshal(bytes, &prs); err != nil { + return relatedPRNotMerged, err + } + + mergeVersionSets := sets.NewString() + for _, pr := range prs { + if pr.State == sdk.StatusMerged { + mergeVersionSets.Insert(pr.Base.Ref) + } + } + + for _, v := range versions { + if !mergeVersionSets.Has(v) { + relatedPRNotMerged = append(relatedPRNotMerged, v) + } + } + + return relatedPRNotMerged, nil +} + +type ResponseData struct { + Code int `json:"code"` + Result CveData `json:"result"` + Msg string `json:"msg"` +} + +type CveData struct { + CveDatabase + SecurityNoticeNo string `json:"securityNoticeNo"` + ParserBean *CveParser `json:"parserBean"` + Cvrf *CveCvrf `json:"cvrf"` + PackageList []CveProductPackage `json:"packageList"` +} + +type CveDatabase struct { + Id int64 `json:"id"` + AffectedProduct string `json:"affectedProduct"` + AnnouncementTime string `json:"announcementTime"` + AttackComplexitynvd string `json:"attackComplexityNVD"` + AttackComplexityoe string `json:"attackComplexityOE"` + AttackVectornvd string `json:"attackVectorNVD"` + AttackVectoroe string `json:"attackVectorOE"` + Availabilitynvd string `json:"availabilityNVD"` + Availabilityoe string `json:"availabilityOE"` + Confidentialitynvd string `json:"confidentialityNVD"` + Confidentialityoe string `json:"confidentialityOE"` + CveId string `json:"cveId"` + CvsssCorenvd string `json:"cvsssCoreNVD"` + CvsssCoreoe string `json:"cvsssCoreOE"` + Integritynvd string `json:"integrityNVD"` + Integrityoe string `json:"integrityOE"` + NationalCyberAwarenessSystem string `json:"nationalCyberAwarenessSystem"` + PackageName string `json:"packageName"` + PrivilegesRequirednvd string `json:"privilegesRequiredNVD"` + PrivilegesRequiredoe string `json:"privilegesRequiredOE"` + Scopenvd string `json:"scopeNVD"` + Scopeoe string `json:"scopeOE"` + Status string `json:"status"` + Summary string `json:"summary"` + Type string `json:"type"` + UserInteractionnvd string `json:"userInteractionNVD"` + UserInteractionoe string `json:"userInteractionOE"` + Updateime string `json:"updateTime"` + CreateTime string `json:"createTime"` +} + +type CveParser struct { + Id int64 `json:"id"` + Cve string `json:"cve"` + Cvss string `json:"cvss"` + Exception string `json:"exception"` + PackageName string `json:"packageName"` + Score string `json:"score"` + SeverityDetail string `json:"severityDetail"` + Vector string `json:"vector"` + Updateime string `json:"updateTime"` +} + +type CveCvrf struct { + Id int64 `json:"id"` + FileName string `json:"fileName"` + CveId string `json:"cveId"` + Cvrf string `json:"cvrf"` + PackageName string `json:"packageName"` + SecurityNoticeNo string `json:"securityNoticeNo"` + Updateime string `json:"updateTime"` +} + +type CveProductPackage struct { + Id int64 `json:"id"` + CveId string `json:"cveId"` + PackageName string `json:"packageName"` + ProductName string `json:"productName"` + Status string `json:"status"` + Reason string `json:"reason"` + SecurityNoticeNo string `json:"securityNoticeNo"` + ReleaseTime string `json:"releaseTime"` + Updateime string `json:"updateTime"` + CreateTime string `json:"createTime"` +} + +// 从cve-sa-backend服务调用接口获取cvedatabase信息 +func getCveByIdAndPackageName(cveId, packageName string) (cve CveData, err error) { + cli := utils.NewHttpClient(defaultClientTimeout) + + url := fmt.Sprintf("%s/cve-security-notice-server/cvedatabase/getByCveIdAndPackageName?cveId=%s&packageName=%s", + endpoint, cveId, packageName) + + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return + } + + r, _, err := cli.Download(request) + if err != nil { + return + } + + var resp ResponseData + if err = json.Unmarshal(r, &resp); err != nil { + return + } + + if resp.Code != 0 { + err = errors.New(resp.Msg) + + return + } + + return resp.Result, nil +} + +// 从cve-sa-backend服务调用接口更新cvedatabase信息 +func updateCveDatabase(cveDatabase CveDatabase) error { + cli := utils.NewHttpClient(defaultClientTimeout) + + url := fmt.Sprintf("%s/cve-security-notice-server/cvedatabase/updateCveDatabase", endpoint) + + requestBody, err := json.Marshal(cveDatabase) + if err != nil { + return err + } + + request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody)) + if err != nil { + return err + } + + request.Header.Set("Content-Type", "application/json") + + _, _, err = cli.Download(request) + + return err +} -- Gitee From 2cf92344425ecace638ce89f51fdd5eac5151387 Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Thu, 26 Dec 2024 19:10:36 +0800 Subject: [PATCH 2/7] fix --- cve-vulner-manager/models/issue.go | 11 ++++--- cve-vulner-manager/task/inittask.go | 8 +++-- cve-vulner-manager/task/issue.go | 48 ++++++++++++++++++++--------- 3 files changed, 45 insertions(+), 22 deletions(-) diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go index 382d331..e1982f0 100644 --- a/cve-vulner-manager/models/issue.go +++ b/cve-vulner-manager/models/issue.go @@ -795,10 +795,13 @@ func (t *IssueTemplate) IsIssueComplete() bool { return t.Status == statusCompleted } -func ListHalfYearCompleteIssues() (issueTemps []IssueTemplate, err error) { - sql := `select * from cve_issue_template where status = %d and create_time > DATE_SUB(NOW(), INTERVAL %d MONTH) and owner = %s` +// ListHalfYearCompleteIssues returns the list of issues completed in the last half year. +func ListHalfYearCompleteIssues() ([]IssueTemplate, error) { + issueTemps := make([]IssueTemplate, 0) + sql := "select * from cve_issue_template where status = %d and create_time > " + + "DATE_SUB(NOW(), INTERVAL %d MONTH) and owner = %s" sql = fmt.Sprintf(sql, statusCompleted, month, owner) o := orm.NewOrm() - _, err = o.Raw(sql).QueryRows(&issueTemps) - return + _, err := o.Raw(sql).QueryRows(&issueTemps) + return issueTemps, err } diff --git a/cve-vulner-manager/task/inittask.go b/cve-vulner-manager/task/inittask.go index a611a94..0b3b12e 100644 --- a/cve-vulner-manager/task/inittask.go +++ b/cve-vulner-manager/task/inittask.go @@ -150,7 +150,7 @@ func PrintLogTask(printLog string) { logs.Info("Execute log task task end...") } -// Statistics of issues created in different communities, uncreated issues are created +// IssueCommunityStatistTask Community issue statistics func IssueCommunityStatistTask(issuecommunity string) { logs.Info("Community issue statistics task started...") issueStatTask := toolbox.NewTask("CommunityIssueStatist", issuecommunity, CommunityIssueStatist) @@ -158,7 +158,7 @@ func IssueCommunityStatistTask(issuecommunity string) { logs.Info("Community issue statistics task is over...") } -// Check whether the issue label is reasonable +// IssueLabelCheckTask Issue label check func IssueLabelCheckTask(issuelabelcheck string) { issueLaCheckTask := toolbox.NewTask("IssueLabelCheck", issuelabelcheck, IssueLabelCheck) toolbox.AddTask("IssueLabelCheck", issueLaCheckTask) @@ -183,8 +183,10 @@ func ReleaseUnaffetcdCveTask(spec string) { toolbox.AddTask("ReleaseUnaffetcdCveTask", toolbox.NewTask("ReleaseUnaffetcdCveTask", spec, ReleaseUnaffectedCve)) } +// DealAnomalousIssueTask deal with abnormal issues func DealAnomalousIssueTask(dealanomalousissue string) { - toolbox.AddTask("DealAnomalousIssueTask", toolbox.NewTask("DealAnomalousIssueTask", dealanomalousissue, DealAnomalousIssues)) + toolbox.AddTask("DealAnomalousIssueTask", + toolbox.NewTask("DealAnomalousIssueTask", dealanomalousissue, DealAnomalousIssues)) } // start task diff --git a/cve-vulner-manager/task/issue.go b/cve-vulner-manager/task/issue.go index 4b3aee6..80b6781 100644 --- a/cve-vulner-manager/task/issue.go +++ b/cve-vulner-manager/task/issue.go @@ -32,6 +32,11 @@ const ( modifiedCvrfTxt = "modified-cvrf.txt" cvssV3 = "v3" statusOpen = "open" + sleepTime = 300 * time.Millisecond + prec = -1 + bitsize = 64 + vectorLenth = 8 + saLenth = 4 ) var ( @@ -423,7 +428,7 @@ func getPlanData() []PlanDataOfMaJun { token := beego.AppConfig.String("majun::api_token") pageNum := 1 pageSize := 1000 - cli := utils.NewHttpClient(3) + cli := utils.NewHttpClient(defaultClientTimeout) var data []PlanDataOfMaJun for { @@ -455,6 +460,7 @@ func getPlanData() []PlanDataOfMaJun { return data } +// DealAnomalousIssues deal with abnormal issues func DealAnomalousIssues() error { defer common.Catchs() @@ -562,7 +568,7 @@ func processIssues(issueTemps []models.IssueTemplate, token string) ( normalIssues = append(normalIssues, issueTemp) // 防止调用giteeAPI太频繁，出现429错误 - time.Sleep(300 * time.Millisecond) + time.Sleep(sleepTime) } return normalIssues, anomalousIssues, anomalousIssueTips @@ -579,11 +585,15 @@ func handleAnomalousIssues(anomalousIssues []models.IssueTemplate, anomalousIssu anomalousIssues[i].Status = 1 anomalousIssues[i].StatusName = statusOpen issueOption := taskhandler.IssueOptions{Token: token, Repo: repo, State: anomalousIssues[i].StatusName} - requestBody, _ := json.Marshal(issueOption) + requestBody, err := json.Marshal(issueOption) + if err != nil { + continue + } + url := "https://gitee.com/api/v5/repos/" + owner + "/issues/" + issueNum // 重新打开issue - _, err := util.HTTPPatch(url, string(requestBody)) + _, err = util.HTTPPatch(url, string(requestBody)) if err != nil { logs.Error("UpdateIssueToGit, Update issue failed, cveNum: ", cveNum, "err: ", err) continue @@ -602,7 +612,7 @@ func handleAnomalousIssues(anomalousIssues []models.IssueTemplate, anomalousIssu // 防止调用giteeAPI太频繁，出现429错误 if i < len(anomalousIssues)-1 { - time.Sleep(300 * time.Millisecond) + time.Sleep(sleepTime) } } } @@ -621,7 +631,7 @@ func handleNormalIssues(normalIssues []models.IssueTemplate) { continue } - eulerScore := strconv.FormatFloat(issue.OpenEulerScore, 'f', -1, 64) + eulerScore := strconv.FormatFloat(issue.OpenEulerScore, 'f', prec, bitsize) if cve.CvsssCoreoe == eulerScore { continue @@ -631,7 +641,7 @@ func handleNormalIssues(normalIssues []models.IssueTemplate) { // 检查len(vectors) == 8 vectors := strings.Split(issue.OpenEulerVector, "/") - if len(vectors) != 8 { + if len(vectors) != vectorLenth { continue } @@ -657,7 +667,7 @@ func handleNormalIssues(normalIssues []models.IssueTemplate) { } securityNoticeNo := strings.Split(cve.SecurityNoticeNo, "-") - if len(securityNoticeNo) != 4 { + if len(securityNoticeNo) != saLenth { continue } @@ -716,7 +726,9 @@ func handleNormalIssues(normalIssues []models.IssueTemplate) { } // 检查issue受影响的分支是否都已经关联PR并且PR都合并 -func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []string) (relatedPRNotMerged []string, err error) { +func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []string) ( + relatedPRNotMerged []string, err error, +) { endpoint := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests?access_token=%s&repo=%s", issueTemp.Owner, issueTemp.IssueNum, token, issueTemp.Repo, ) @@ -726,7 +738,7 @@ func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []str } var prs []sdk.PullRequest - cli := utils.NewHttpClient(3) + cli := utils.NewHttpClient(defaultClientTimeout) bytes, _, err := cli.Download(req) if err != nil { return relatedPRNotMerged, err @@ -752,12 +764,14 @@ func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []str return relatedPRNotMerged, nil } +// ResponseData respresent the response data type ResponseData struct { Code int `json:"code"` Result CveData `json:"result"` Msg string `json:"msg"` } +// CveData respresent the cve data type CveData struct { CveDatabase SecurityNoticeNo string `json:"securityNoticeNo"` @@ -766,6 +780,7 @@ type CveData struct { PackageList []CveProductPackage `json:"packageList"` } +// CveDatabase respresent the cve database type CveDatabase struct { Id int64 `json:"id"` AffectedProduct string `json:"affectedProduct"` @@ -798,6 +813,7 @@ type CveDatabase struct { CreateTime string `json:"createTime"` } +// CveParser respresent the cve parser type CveParser struct { Id int64 `json:"id"` Cve string `json:"cve"` @@ -810,6 +826,7 @@ type CveParser struct { Updateime string `json:"updateTime"` } +// CveCvrf respresent the cve cvrf type CveCvrf struct { Id int64 `json:"id"` FileName string `json:"fileName"` @@ -820,6 +837,7 @@ type CveCvrf struct { Updateime string `json:"updateTime"` } +// CveProductPackage respresent the cve product package type CveProductPackage struct { Id int64 `json:"id"` CveId string `json:"cveId"` @@ -834,7 +852,7 @@ type CveProductPackage struct { } // 从cve-sa-backend服务调用接口获取cvedatabase信息 -func getCveByIdAndPackageName(cveId, packageName string) (cve CveData, err error) { +func getCveByIdAndPackageName(cveId, packageName string) (CveData, error) { cli := utils.NewHttpClient(defaultClientTimeout) url := fmt.Sprintf("%s/cve-security-notice-server/cvedatabase/getByCveIdAndPackageName?cveId=%s&packageName=%s", @@ -842,23 +860,23 @@ func getCveByIdAndPackageName(cveId, packageName string) (cve CveData, err error request, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { - return + return CveData{}, err } r, _, err := cli.Download(request) if err != nil { - return + return CveData{}, err } var resp ResponseData if err = json.Unmarshal(r, &resp); err != nil { - return + return CveData{}, err } if resp.Code != 0 { err = errors.New(resp.Msg) - return + return CveData{}, err } return resp.Result, nil -- Gitee From 4fd9f7bd10773b44a59fb71d5287aef0acee27ff Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Fri, 27 Dec 2024 10:41:44 +0800 Subject: [PATCH 3/7] fix --- cve-vulner-manager/task/issue.go | 272 +++++++++++++++++-------------- 1 file changed, 146 insertions(+), 126 deletions(-) diff --git a/cve-vulner-manager/task/issue.go b/cve-vulner-manager/task/issue.go index 80b6781..e4310b5 100644 --- a/cve-vulner-manager/task/issue.go +++ b/cve-vulner-manager/task/issue.go @@ -13,7 +13,7 @@ import ( "github.com/astaxie/beego" "github.com/astaxie/beego/logs" - sdk "github.com/opensourceways/go-gitee/gitee" + "github.com/opensourceways/go-gitee/gitee" "github.com/opensourceways/server-common-lib/utils" "k8s.io/apimachinery/pkg/util/sets" @@ -500,68 +500,12 @@ func processIssues(issueTemps []models.IssueTemplate, token string) ( continue } - score, err := models.QueryIssueScore(issueTemp.CveId) - if err != nil { - logs.Error("query issue score error: ", err.Error()) - continue - } - - // 如果score type 为 v3, 则检查 cvss v3 vector 和 score 是否一致 - if score.ScoreType == cvssV3 { - if util.CalculateCVSSV3BaseScore(issueTemp.OpenEulerVector) != issueTemp.OpenEulerScore { - tip := fmt.Sprintf("@%v cvss v3 vector: %v and score: %f are not consistent", - issueTemp.Assignee, issueTemp.OpenEulerVector, issueTemp.OpenEulerScore) - + tip, ok := checkIssueTemp(issueTemp, token) + if !ok { + if tip != "" { anomalousIssues = append(anomalousIssues, issueTemp) anomalousIssueTips[issueTemp.TemplateId] = tip - continue - } - } - - cveCenter := models.VulnCenter{CveId: issueTemp.CveId, CveNum: issueTemp.CveNum} - err = models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") - if err != nil { - logs.Error("get vuln center error: ", err.Error()) - continue - } - - // 检查issue的影响性分析是否完成 - if msg, _, ok := taskhandler.CheckIssueAnalysisComplete(&issueTemp, cveCenter.OrganizationID); !ok { - na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" - tip := fmt.Sprintf("@%v %v", issueTemp.Assignee, msg) + na - - anomalousIssues = append(anomalousIssues, issueTemp) - anomalousIssueTips[issueTemp.TemplateId] = tip - continue - } - - // 检查issue的受影响分支关联的PR是否合入 - versions := strings.Split(issueTemp.AffectedVersion, ",") - affectedVersions := make([]string, 0) - for _, v := range versions { - if !strings.Contains(v, "不受影响") { - affectedVersions = append(affectedVersions, v) } - } - - relatedPRNotMerged, err := checkRelatedPR(issueTemp, token, affectedVersions) - if err != nil { - logs.Error("check related PR error: ", err.Error()) - continue - } - - if len(relatedPRNotMerged) > 0 { - tip := fmt.Sprintf("@%s\n"+ - "关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #%s\n"+ - "受影响分支: %s\n"+ - "具体操作参考: %s\n", - issueTemp.Assignee, - issueTemp.IssueNum, - strings.Join(relatedPRNotMerged, "/"), - referenceUrl) - - anomalousIssues = append(anomalousIssues, issueTemp) - anomalousIssueTips[issueTemp.TemplateId] = tip continue } @@ -637,78 +581,22 @@ func handleNormalIssues(normalIssues []models.IssueTemplate) { continue } - cve.CvsssCoreoe = eulerScore - - // 检查len(vectors) == 8 - vectors := strings.Split(issue.OpenEulerVector, "/") - if len(vectors) != vectorLenth { - continue - } - - cve.AttackVectoroe = mapVector[vectors[0]] - cve.AttackComplexityoe = mapVector[vectors[1]] - cve.PrivilegesRequiredoe = mapVector[vectors[2]] - cve.UserInteractionoe = mapVector[vectors[3]] - cve.Scopeoe = mapVector[vectors[4]] - cve.Confidentialityoe = mapVector[vectors[5]] - cve.Integrityoe = mapVector[vectors[6]] - cve.Availabilityoe = mapVector[vectors[7]] - - // 更新cvedatabase - err = updateCveDatabase(cve.CveDatabase) - if err != nil { - logs.Error("updateCveDatabase error: ", err.Error(), "cveDatabase: ", cve.CveDatabase.CveId) + // 更新cve官网评分 + if err = modifiyCve(cve, issue, eulerScore); err != nil { + logs.Error("modifiyCve error: ", err.Error(), + "cveNum: ", issue.CveNum, "repo: ", issue.Repo) continue } - // 修改cvrf文件 if cve.SecurityNoticeNo == "" { continue } - securityNoticeNo := strings.Split(cve.SecurityNoticeNo, "-") - if len(securityNoticeNo) != saLenth { - continue - } - - year := securityNoticeNo[2] - - cvrfFilePath := fmt.Sprintf("%s%s/%s", cvrfdir, year, cve.SecurityNoticeNo) - - content, err := obs.Download(cvrfFilePath) - if err != nil { - logs.Error("download cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfFilePath) - continue - } - - var cvrf bulletinimpl.Cvrf - err = xml.Unmarshal(content, &cvrf) - if err != nil { - logs.Error("unmarshal cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfFilePath) - continue - } - - for i := range cvrf.Vulnerability { - if cvrf.Vulnerability[i].CVE == issue.CveNum { - cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.BaseScore = eulerScore - cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.Vector = issue.OpenEulerVector - - break - } - } - - uploadBys, err := xml.MarshalIndent(cvrf, "", " ") - if err != nil { - logs.Error("marshal cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfFilePath) - continue - } - - headerBytes := []byte(xml.Header) - headerBytes = append(headerBytes, uploadBys...) - - err = obs.Upload(cvrfFilePath, headerBytes) + // 修改cvrf文件 + cvrfFilePath, err := modifyCvrfFile(cve, issue, eulerScore, cvrfdir) if err != nil { - logs.Error("upload cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfFilePath) + logs.Error("modifyCvrfFile error: ", err.Error(), + "cveNum: ", issue.CveNum, "repo: ", issue.Repo) continue } @@ -737,7 +625,7 @@ func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []str return relatedPRNotMerged, err } - var prs []sdk.PullRequest + var prs []gitee.PullRequest cli := utils.NewHttpClient(defaultClientTimeout) bytes, _, err := cli.Download(req) if err != nil { @@ -750,7 +638,7 @@ func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []str mergeVersionSets := sets.NewString() for _, pr := range prs { - if pr.State == sdk.StatusMerged { + if pr.State == gitee.StatusMerged { mergeVersionSets.Insert(pr.Base.Ref) } } @@ -904,3 +792,135 @@ func updateCveDatabase(cveDatabase CveDatabase) error { return err } + +func checkIssueTemp(issueTemp models.IssueTemplate, token string) (string, bool) { + if issueTemp.OpenEulerScore <= 0 { + return "openeuler评分不能为0！", false + } + + score, err := models.QueryIssueScore(issueTemp.CveId) + if err != nil { + logs.Error("query issue score error: ", err.Error()) + return "", false + } + + // 如果score type 为 v3, 则检查 cvss v3 vector 和 score 是否一致 + if score.ScoreType == cvssV3 { + if util.CalculateCVSSV3BaseScore(issueTemp.OpenEulerVector) != issueTemp.OpenEulerScore { + tip := fmt.Sprintf("@%v cvss v3 vector: %v and score: %f are not consistent", + issueTemp.Assignee, issueTemp.OpenEulerVector, issueTemp.OpenEulerScore) + + return tip, false + } + } + + cveCenter := models.VulnCenter{CveId: issueTemp.CveId, CveNum: issueTemp.CveNum} + err = models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") + if err != nil { + logs.Error("get vuln center error: ", err.Error()) + return "", false + } + + // 检查issue的影响性分析是否完成 + if msg, _, ok := taskhandler.CheckIssueAnalysisComplete(&issueTemp, cveCenter.OrganizationID); !ok { + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + tip := fmt.Sprintf("@%v %v", issueTemp.Assignee, msg) + na + + return tip, false + } + + // 检查issue的受影响分支关联的PR是否合入 + versions := strings.Split(issueTemp.AffectedVersion, ",") + affectedVersions := make([]string, 0) + for _, v := range versions { + if !strings.Contains(v, "不受影响") { + affectedVersions = append(affectedVersions, v) + } + } + + relatedPRNotMerged, err := checkRelatedPR(issueTemp, token, affectedVersions) + if err != nil { + logs.Error("check related PR error: ", err.Error()) + return "", false + } + + if len(relatedPRNotMerged) > 0 { + tip := fmt.Sprintf("@%s\n"+ + "关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #%s\n"+ + "受影响分支: %s\n"+ + "具体操作参考: %s\n", + issueTemp.Assignee, issueTemp.IssueNum, strings.Join(relatedPRNotMerged, "/"), referenceUrl) + + return tip, false + } + + return "", true +} + +func modifiyCve(cve CveData, issue models.IssueTemplate, eulerScore string) error { + cve.CvsssCoreoe = eulerScore + + // 检查len(vectors) == 8 + vectors := strings.Split(issue.OpenEulerVector, "/") + if len(vectors) != vectorLenth { + return errors.New("invalid openeuler vector") + } + + cve.AttackVectoroe = mapVector[vectors[0]] + cve.AttackComplexityoe = mapVector[vectors[1]] + cve.PrivilegesRequiredoe = mapVector[vectors[2]] + cve.UserInteractionoe = mapVector[vectors[3]] + cve.Scopeoe = mapVector[vectors[4]] + cve.Confidentialityoe = mapVector[vectors[5]] + cve.Integrityoe = mapVector[vectors[6]] + cve.Availabilityoe = mapVector[vectors[7]] + + // 更新cvedatabase + return updateCveDatabase(cve.CveDatabase) +} + +func modifyCvrfFile(cve CveData, issue models.IssueTemplate, eulerScore, cvrfdir string) (string, error) { + obs := obsimpl.Instance() + securityNoticeNo := strings.Split(cve.SecurityNoticeNo, "-") + if len(securityNoticeNo) != saLenth { + return "", errors.New("invalid security notice no") + } + + year := securityNoticeNo[2] + + cvrfFilePath := fmt.Sprintf("%s%s/%s", cvrfdir, year, cve.SecurityNoticeNo) + + content, err := obs.Download(cvrfFilePath) + if err != nil { + return "", err + } + + var cvrf bulletinimpl.Cvrf + err = xml.Unmarshal(content, &cvrf) + if err != nil { + return "", err + } + + for i := range cvrf.Vulnerability { + if cvrf.Vulnerability[i].CVE == issue.CveNum { + cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.BaseScore = eulerScore + cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.Vector = issue.OpenEulerVector + + break + } + } + + uploadBys, err := xml.MarshalIndent(cvrf, "", " ") + if err != nil { + return "", err + } + + headerBytes := []byte(xml.Header) + headerBytes = append(headerBytes, uploadBys...) + + if err = obs.Upload(cvrfFilePath, headerBytes); err != nil { + return "", err + } + + return cvrfFilePath, nil +} -- Gitee From fd1dc12af2785edca19604b2688dd827ff28c97a Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Thu, 2 Jan 2025 19:19:36 +0800 Subject: [PATCH 4/7] fix review code --- cve-vulner-manager/models/issue.go | 9 +- cve-vulner-manager/task/anomalousissuetask.go | 541 ++++++++++++++++++ cve-vulner-manager/task/issue.go | 473 --------------- 3 files changed, 545 insertions(+), 478 deletions(-) create mode 100644 cve-vulner-manager/task/anomalousissuetask.go diff --git a/cve-vulner-manager/models/issue.go b/cve-vulner-manager/models/issue.go index e1982f0..ddc37e2 100644 --- a/cve-vulner-manager/models/issue.go +++ b/cve-vulner-manager/models/issue.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "sync" + "time" "cvevulner/common" "cvevulner/util" @@ -796,12 +797,10 @@ func (t *IssueTemplate) IsIssueComplete() bool { } // ListHalfYearCompleteIssues returns the list of issues completed in the last half year. -func ListHalfYearCompleteIssues() ([]IssueTemplate, error) { +func ListHalfYearCompleteIssues(createdTime time.Time) ([]IssueTemplate, error) { issueTemps := make([]IssueTemplate, 0) - sql := "select * from cve_issue_template where status = %d and create_time > " + - "DATE_SUB(NOW(), INTERVAL %d MONTH) and owner = %s" - sql = fmt.Sprintf(sql, statusCompleted, month, owner) o := orm.NewOrm() - _, err := o.Raw(sql).QueryRows(&issueTemps) + _, err := o.Raw("select * from cve_issue_template where status = ? and create_time > ? and owner = ?", + statusCompleted, createdTime, owner).QueryRows(&issueTemps) return issueTemps, err } diff --git a/cve-vulner-manager/task/anomalousissuetask.go b/cve-vulner-manager/task/anomalousissuetask.go new file mode 100644 index 0000000..9d211f8 --- /dev/null +++ b/cve-vulner-manager/task/anomalousissuetask.go @@ -0,0 +1,541 @@ +package task + +import ( + "bytes" + "cvevulner/common" + "cvevulner/cve-ddd/infrastructure/bulletinimpl" + "cvevulner/cve-ddd/infrastructure/obsimpl" + "cvevulner/models" + "cvevulner/taskhandler" + "cvevulner/util" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "github.com/opensourceways/go-gitee/gitee" + "github.com/opensourceways/server-common-lib/utils" + "k8s.io/apimachinery/pkg/util/sets" +) + +const month = 6 + +// DealAnomalousIssues deal with abnormal issues +func DealAnomalousIssues() error { + defer common.Catchs() + + createdTime := time.Now().AddDate(0, -month, 0) + issueTemps, issueErr := models.ListHalfYearCompleteIssues(createdTime) + if issueErr != nil { + return issueErr + } + + if len(issueTemps) == 0 { + logs.Info("there are no issues that need to be processed") + return nil + } + + _, token := common.GetOwnerAndToken("", 1) + + normalIssues, anomalousIssues, anomalousIssueTips := processIssues(issueTemps, token) + + handleAnomalousIssues(anomalousIssues, anomalousIssueTips, token) + + handleNormalIssues(normalIssues) + + return nil +} + +// 筛选出不满足关闭条件的issue，这种issue需要重新打开并评论提示 +// 满足关闭条件但issue实际没有关闭的情况基本没有，所以不考虑处理这种issue +func processIssues(issueTemps []models.IssueTemplate, token string) ( + []models.IssueTemplate, + []models.IssueTemplate, + map[int64]string, +) { + normalIssues := make([]models.IssueTemplate, 0) + anomalousIssues := make([]models.IssueTemplate, 0) + anomalousIssueTips := make(map[int64]string) + + for _, issueTemp := range issueTemps { + tip, err := checkIssueTemp(issueTemp, token) + if err != nil { + logs.Error(err) + + continue + } + + if tip != "" { + anomalousIssues = append(anomalousIssues, issueTemp) + anomalousIssueTips[issueTemp.TemplateId] = tip + + continue + } + + normalIssues = append(normalIssues, issueTemp) + + // 防止调用giteeAPI太频繁，出现429错误 + time.Sleep(sleepTime) + } + + return normalIssues, anomalousIssues, anomalousIssueTips +} + +// 与gitee平台进行交互，重新打开异常关闭的issue并评论提示 +func handleAnomalousIssues(anomalousIssues []models.IssueTemplate, anomalousIssueTips map[int64]string, token string) { + for i := range anomalousIssues { + issueNum := anomalousIssues[i].IssueNum + owner := anomalousIssues[i].Owner + repo := anomalousIssues[i].Repo + cveNum := anomalousIssues[i].CveNum + + anomalousIssues[i].Status = 1 + anomalousIssues[i].StatusName = statusOpen + issueOption := taskhandler.IssueOptions{Token: token, Repo: repo, State: anomalousIssues[i].StatusName} + requestBody, err := json.Marshal(issueOption) + if err != nil { + continue + } + + url := "https://gitee.com/api/v5/repos/" + owner + "/issues/" + issueNum + + // 重新打开issue + _, err = util.HTTPPatch(url, string(requestBody)) + if err != nil { + logs.Error("UpdateIssueToGit, Update issue failed, cveNum: ", cveNum, "err: ", err) + continue + } + + // 评论提示相关错误信息 + tip := anomalousIssueTips[anomalousIssues[i].TemplateId] + taskhandler.AddCommentToIssue(tip, issueNum, owner, repo, token) + + // 更新数据库表CveIssueTemplate + err = models.UpdateIssueTemplate(&anomalousIssues[i], "Status", "StatusName") + if err != nil { + logs.Error("UpdateIssueTemplate, Update issue failed, cveNum: ", cveNum, "err: ", err) + continue + } + + // 防止调用giteeAPI太频繁，出现429错误 + time.Sleep(sleepTime) + } +} + +// 正常关闭的issue需要校验openeuler评分是否和cve官网一致，如果不一致，需要更新cve官网评分和cvrf文件内容 +func handleNormalIssues(normalIssues []models.IssueTemplate) { + obs := obsimpl.Instance() + cvrfdir := beego.AppConfig.String("obs::download_cvrf_dir") + modifiedCvrfFiles := sets.NewString() + + for _, issue := range normalIssues { + cve, err := getCveByIdAndPackageName(issue.CveNum, issue.Repo) + if err != nil { + logs.Error("getCveByIdAndPackageName error: ", err.Error(), + "cveNum: ", issue.CveNum, "repo: ", issue.Repo) + continue + } + + eulerScore := strconv.FormatFloat(issue.OpenEulerScore, 'f', prec, bitsize) + + if cve.CvsssCoreoe == eulerScore { + continue + } + + // 更新cve官网评分 + if err = modifiyCve(cve, issue, eulerScore); err != nil { + logs.Error("modifiyCve error: ", err.Error(), + "cveNum: ", issue.CveNum, "repo: ", issue.Repo) + continue + } + + if cve.SecurityNoticeNo == "" { + continue + } + + // 修改cvrf文件 + cvrfFilePath, err := modifyCvrfFile(cve, issue, eulerScore, cvrfdir) + if err != nil { + logs.Error("modifyCvrfFile error: ", err.Error(), + "cveNum: ", issue.CveNum, "repo: ", issue.Repo) + continue + } + + modifiedCvrfFiles.Insert(cvrfFilePath...) + } + + if len(modifiedCvrfFiles) > 0 { + content := strings.Join(modifiedCvrfFiles.List(), "\n") + + err := obs.Upload(cvrfdir+modifiedCvrfTxt, []byte(content)) + if err != nil { + logs.Error("upload cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfdir+modifiedCvrfTxt) + } + } +} + +// 检查issue受影响的分支是否都已经关联PR并且PR都合并 +func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []string) ( + relatedPRNotMerged []string, err error, +) { + endpoint := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests?access_token=%s&repo=%s", + issueTemp.Owner, issueTemp.IssueNum, token, issueTemp.Repo, + ) + req, err := http.NewRequest(http.MethodGet, endpoint, nil) + if err != nil { + return relatedPRNotMerged, err + } + + var prs []gitee.PullRequest + cli := utils.NewHttpClient(defaultClientTimeout) + bytes, _, err := cli.Download(req) + if err != nil { + return relatedPRNotMerged, err + } + + if err := json.Unmarshal(bytes, &prs); err != nil { + return relatedPRNotMerged, err + } + + mergeVersionSets := sets.NewString() + for _, pr := range prs { + if pr.State == gitee.StatusMerged { + mergeVersionSets.Insert(pr.Base.Ref) + } + } + + for _, v := range versions { + if !mergeVersionSets.Has(v) { + relatedPRNotMerged = append(relatedPRNotMerged, v) + } + } + + return relatedPRNotMerged, nil +} + +// ResponseData respresent the response data +type ResponseCveData struct { + Result CveData `json:"result"` + CommonData +} + +type ResponseCveProductPackage struct { + Result []CveProductPackage `json:"result"` + CommonData +} + +// ResponseData respresent the response data +type CommonData struct { + Code int `json:"code"` + Msg string `json:"msg"` +} + +// CveData respresent the cve data +type CveData struct { + CveDatabase + SecurityNoticeNo string `json:"securityNoticeNo"` + ParserBean *CveParser `json:"parserBean"` + Cvrf *CveCvrf `json:"cvrf"` + PackageList []CveProductPackage `json:"packageList"` +} + +// CveDatabase respresent the cve database +type CveDatabase struct { + Id int64 `json:"id"` + AffectedProduct string `json:"affectedProduct"` + AnnouncementTime string `json:"announcementTime"` + AttackComplexitynvd string `json:"attackComplexityNVD"` + AttackComplexityoe string `json:"attackComplexityOE"` + AttackVectornvd string `json:"attackVectorNVD"` + AttackVectoroe string `json:"attackVectorOE"` + Availabilitynvd string `json:"availabilityNVD"` + Availabilityoe string `json:"availabilityOE"` + Confidentialitynvd string `json:"confidentialityNVD"` + Confidentialityoe string `json:"confidentialityOE"` + CveId string `json:"cveId"` + CvsssCorenvd string `json:"cvsssCoreNVD"` + CvsssCoreoe string `json:"cvsssCoreOE"` + Integritynvd string `json:"integrityNVD"` + Integrityoe string `json:"integrityOE"` + NationalCyberAwarenessSystem string `json:"nationalCyberAwarenessSystem"` + PackageName string `json:"packageName"` + PrivilegesRequirednvd string `json:"privilegesRequiredNVD"` + PrivilegesRequiredoe string `json:"privilegesRequiredOE"` + Scopenvd string `json:"scopeNVD"` + Scopeoe string `json:"scopeOE"` + Status string `json:"status"` + Summary string `json:"summary"` + Type string `json:"type"` + UserInteractionnvd string `json:"userInteractionNVD"` + UserInteractionoe string `json:"userInteractionOE"` + Updateime string `json:"updateTime"` + CreateTime string `json:"createTime"` +} + +// CveParser respresent the cve parser +type CveParser struct { + Id int64 `json:"id"` + Cve string `json:"cve"` + Cvss string `json:"cvss"` + Exception string `json:"exception"` + PackageName string `json:"packageName"` + Score string `json:"score"` + SeverityDetail string `json:"severityDetail"` + Vector string `json:"vector"` + Updateime string `json:"updateTime"` +} + +// CveCvrf respresent the cve cvrf +type CveCvrf struct { + Id int64 `json:"id"` + FileName string `json:"fileName"` + CveId string `json:"cveId"` + Cvrf string `json:"cvrf"` + PackageName string `json:"packageName"` + SecurityNoticeNo string `json:"securityNoticeNo"` + Updateime string `json:"updateTime"` +} + +// CveProductPackage respresent the cve product package +type CveProductPackage struct { + Id int64 `json:"id"` + CveId string `json:"cveId"` + PackageName string `json:"packageName"` + ProductName string `json:"productName"` + Status string `json:"status"` + Reason string `json:"reason"` + SecurityNoticeNo string `json:"securityNoticeNo"` + ReleaseTime string `json:"releaseTime"` + Updateime string `json:"updateTime"` + CreateTime time.Time `json:"createTime"` +} + +// 从cve-sa-backend服务调用接口获取cvedatabase信息 +func getCveByIdAndPackageName(cveId, packageName string) (CveData, error) { + var resp ResponseCveData + url := fmt.Sprintf("%s/cve-security-notice-server/cvedatabase/getByCveIdAndPackageName?cveId=%s&packageName=%s", + endpoint, cveId, packageName) + + if err := getDataFromEndpoint(url, &resp); err != nil { + return CveData{}, err + } + + if resp.Code != 0 { + return CveData{}, errors.New(resp.Msg) + } + + return resp.Result, nil +} + +func getCVEProductPackageList(cveId, packageName string) ([]CveProductPackage, error) { + var resp ResponseCveProductPackage + url := fmt.Sprintf("%s/cve-security-notice-server/cvedatabase/getCVEProductPackageList?cveId=%s&packageName=%s", + endpoint, cveId, packageName) + + if err := getDataFromEndpoint(url, &resp); err != nil { + return nil, err + } + + if resp.Code != 0 { + return nil, errors.New(resp.Msg) + } + + return resp.Result, nil +} + +func getDataFromEndpoint(url string, resp interface{}) error { + cli := utils.NewHttpClient(defaultClientTimeout) + + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return err + } + + r, _, err := cli.Download(request) + if err != nil { + return err + } + + if err := json.Unmarshal(r, resp); err != nil { + return err + } + + return nil +} + +// 从cve-sa-backend服务调用接口更新cvedatabase信息 +func updateCveDatabase(cveDatabase CveDatabase) error { + cli := utils.NewHttpClient(defaultClientTimeout) + + url := fmt.Sprintf("%s/cve-security-notice-server/cvedatabase/updateCveDatabase", endpoint) + + requestBody, err := json.Marshal(cveDatabase) + if err != nil { + return err + } + + request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody)) + if err != nil { + return err + } + + request.Header.Set("Content-Type", "application/json") + + _, _, err = cli.Download(request) + + return err +} + +func checkIssueTemp(issueTemp models.IssueTemplate, token string) (string, error) { + if issueTemp.OpenEulerScore <= 0 { + return "openeuler评分不能为0！", nil + } + + score, err := models.QueryIssueScore(issueTemp.CveId) + if err != nil { + return "", fmt.Errorf("query issue score error: %w", err) + } + + // 如果score type 为 v3, 则检查 cvss v3 vector 和 score 是否一致 + if score.ScoreType == cvssV3 { + if util.CalculateCVSSV3BaseScore(issueTemp.OpenEulerVector) != issueTemp.OpenEulerScore { + tip := fmt.Sprintf("@%v openEuler 评分和向量不一致！", issueTemp.Assignee) + + return tip, nil + } + } + + cveCenter := models.VulnCenter{CveId: issueTemp.CveId, CveNum: issueTemp.CveNum} + err = models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") + if err != nil { + return "", fmt.Errorf("get vuln center error: %w", err) + } + + // 检查issue的影响性分析是否完成 + if msg, _, ok := taskhandler.CheckIssueAnalysisComplete(&issueTemp, cveCenter.OrganizationID); !ok { + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + tip := fmt.Sprintf("@%v %v", issueTemp.Assignee, msg) + na + + return tip, nil + } + + // 检查issue的受影响分支关联的PR是否合入 + versions := strings.Split(issueTemp.AffectedVersion, ",") + affectedVersions := make([]string, 0) + for _, v := range versions { + if !strings.Contains(v, "不受影响") { + affectedVersions = append(affectedVersions, v) + } + } + + relatedPRNotMerged, err := checkRelatedPR(issueTemp, token, affectedVersions) + if err != nil { + return "", fmt.Errorf("check related pr error: %w", err) + } + + if len(relatedPRNotMerged) > 0 { + tip := fmt.Sprintf("@%s\n"+ + "关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #%s\n"+ + "受影响分支: %s\n"+ + "具体操作参考: %s\n", + issueTemp.Assignee, issueTemp.IssueNum, strings.Join(relatedPRNotMerged, "/"), referenceUrl) + + return tip, nil + } + + return "", nil +} + +func modifiyCve(cve CveData, issue models.IssueTemplate, eulerScore string) error { + cve.CvsssCoreoe = eulerScore + + // 检查len(vectors) == 8 + vectors := strings.Split(issue.OpenEulerVector, "/") + if len(vectors) != vectorLenth { + return errors.New("invalid openeuler vector") + } + + cve.AttackVectoroe = mapVector[vectors[0]] + cve.AttackComplexityoe = mapVector[vectors[1]] + cve.PrivilegesRequiredoe = mapVector[vectors[2]] + cve.UserInteractionoe = mapVector[vectors[3]] + cve.Scopeoe = mapVector[vectors[4]] + cve.Confidentialityoe = mapVector[vectors[5]] + cve.Integrityoe = mapVector[vectors[6]] + cve.Availabilityoe = mapVector[vectors[7]] + + // 更新cvedatabase + return updateCveDatabase(cve.CveDatabase) +} + +func modifyCvrfFile(cve CveData, issue models.IssueTemplate, eulerScore, cvrfdir string) ([]string, error) { + obs := obsimpl.Instance() + + cveProductPackages, err := getCVEProductPackageList(issue.CveNum, issue.Repo) + if err != nil { + return nil, err + } + + securityNoticeNos := sets.NewString() + for _, cveProductPackage := range cveProductPackages { + if cveProductPackage.SecurityNoticeNo != "" { + securityNoticeNos.Insert(cveProductPackage.SecurityNoticeNo) + } + } + + cvrfFiles := sets.NewString() + securityNoticeSlice := securityNoticeNos.List() + + for _, s := range securityNoticeSlice { + securityNoticeNo := strings.Split(s, "-") + if len(securityNoticeNo) != saLenth { + return nil, errors.New("invalid security notice no") + } + + year := securityNoticeNo[2] + + cvrfFilePath := fmt.Sprintf("%s%s/%s", cvrfdir, year, cve.SecurityNoticeNo) + + content, err := obs.Download(cvrfFilePath) + if err != nil { + return nil, err + } + + var cvrf bulletinimpl.Cvrf + err = xml.Unmarshal(content, &cvrf) + if err != nil { + return nil, err + } + + for i := range cvrf.Vulnerability { + if cvrf.Vulnerability[i].CVE == issue.CveNum { + cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.BaseScore = eulerScore + cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.Vector = issue.OpenEulerVector + + break + } + } + + uploadBys, err := xml.MarshalIndent(cvrf, "", " ") + if err != nil { + return nil, err + } + + headerBytes := []byte(xml.Header) + headerBytes = append(headerBytes, uploadBys...) + + if err = obs.Upload(cvrfFilePath, headerBytes); err != nil { + return nil, err + } + + cvrfFiles.Insert(cvrfFilePath) + } + + return cvrfFiles.List(), nil +} diff --git a/cve-vulner-manager/task/issue.go b/cve-vulner-manager/task/issue.go index e4310b5..25b70b9 100644 --- a/cve-vulner-manager/task/issue.go +++ b/cve-vulner-manager/task/issue.go @@ -3,9 +3,6 @@ package task import ( "bytes" "encoding/json" - "encoding/xml" - "errors" - "fmt" "net/http" "strconv" "strings" @@ -13,16 +10,11 @@ import ( "github.com/astaxie/beego" "github.com/astaxie/beego/logs" - "github.com/opensourceways/go-gitee/gitee" "github.com/opensourceways/server-common-lib/utils" - "k8s.io/apimachinery/pkg/util/sets" "cvevulner/common" - "cvevulner/cve-ddd/infrastructure/bulletinimpl" - "cvevulner/cve-ddd/infrastructure/obsimpl" "cvevulner/models" "cvevulner/taskhandler" - "cvevulner/util" ) const ( @@ -459,468 +451,3 @@ func getPlanData() []PlanDataOfMaJun { return data } - -// DealAnomalousIssues deal with abnormal issues -func DealAnomalousIssues() error { - defer common.Catchs() - - issueTemps, issueErr := models.ListHalfYearCompleteIssues() - if len(issueTemps) == 0 { - logs.Error("list half year created and status is complete issues occurred err : ", issueErr) - return issueErr - } - - _, token := common.GetOwnerAndToken("", 1) - - normalIssues, anomalousIssues, anomalousIssueTips := processIssues(issueTemps, token) - - handleAnomalousIssues(anomalousIssues, anomalousIssueTips, token) - - handleNormalIssues(normalIssues) - - return nil -} - -// 筛选出不满足关闭条件的issue，这种issue需要重新打开并评论提示 -// 满足关闭条件但issue实际没有关闭的情况基本没有，所以不考虑处理这种issue -func processIssues(issueTemps []models.IssueTemplate, token string) ( - []models.IssueTemplate, - []models.IssueTemplate, - map[int64]string, -) { - normalIssues := make([]models.IssueTemplate, 0) - anomalousIssues := make([]models.IssueTemplate, 0) - anomalousIssueTips := make(map[int64]string) - - for _, issueTemp := range issueTemps { - if issueTemp.OpenEulerScore <= 0 { - tip := "openeuler评分不能为0！" - anomalousIssues = append(anomalousIssues, issueTemp) - anomalousIssueTips[issueTemp.TemplateId] = tip - continue - } - - tip, ok := checkIssueTemp(issueTemp, token) - if !ok { - if tip != "" { - anomalousIssues = append(anomalousIssues, issueTemp) - anomalousIssueTips[issueTemp.TemplateId] = tip - } - continue - } - - normalIssues = append(normalIssues, issueTemp) - - // 防止调用giteeAPI太频繁，出现429错误 - time.Sleep(sleepTime) - } - - return normalIssues, anomalousIssues, anomalousIssueTips -} - -// 与gitee平台进行交互，重新打开异常关闭的issue并评论提示 -func handleAnomalousIssues(anomalousIssues []models.IssueTemplate, anomalousIssueTips map[int64]string, token string) { - for i := range anomalousIssues { - issueNum := anomalousIssues[i].IssueNum - owner := anomalousIssues[i].Owner - repo := anomalousIssues[i].Repo - cveNum := anomalousIssues[i].CveNum - - anomalousIssues[i].Status = 1 - anomalousIssues[i].StatusName = statusOpen - issueOption := taskhandler.IssueOptions{Token: token, Repo: repo, State: anomalousIssues[i].StatusName} - requestBody, err := json.Marshal(issueOption) - if err != nil { - continue - } - - url := "https://gitee.com/api/v5/repos/" + owner + "/issues/" + issueNum - - // 重新打开issue - _, err = util.HTTPPatch(url, string(requestBody)) - if err != nil { - logs.Error("UpdateIssueToGit, Update issue failed, cveNum: ", cveNum, "err: ", err) - continue - } - - // 更新数据库表CveIssueTemplate - err = models.UpdateIssueTemplate(&anomalousIssues[i], "Status", "StatusName") - if err != nil { - logs.Error("UpdateIssueTemplate, Update issue failed, cveNum: ", cveNum, "err: ", err) - continue - } - - // 评论提示相关错误信息 - tip := anomalousIssueTips[anomalousIssues[i].TemplateId] - taskhandler.AddCommentToIssue(tip, issueNum, owner, repo, token) - - // 防止调用giteeAPI太频繁，出现429错误 - if i < len(anomalousIssues)-1 { - time.Sleep(sleepTime) - } - } -} - -// 正常关闭的issue需要校验openeuler评分是否和cve官网一致，如果不一致，需要更新cve官网评分和cvrf文件内容 -func handleNormalIssues(normalIssues []models.IssueTemplate) { - obs := obsimpl.Instance() - cvrfdir := beego.AppConfig.String("obs::download_cvrf_dir") - modifiedCvrfFiles := make([]string, 0) - - for _, issue := range normalIssues { - cve, err := getCveByIdAndPackageName(issue.CveNum, issue.Repo) - if err != nil { - logs.Error("getCveByIdAndPackageName error: ", err.Error(), - "cveNum: ", issue.CveNum, "repo: ", issue.Repo) - continue - } - - eulerScore := strconv.FormatFloat(issue.OpenEulerScore, 'f', prec, bitsize) - - if cve.CvsssCoreoe == eulerScore { - continue - } - - // 更新cve官网评分 - if err = modifiyCve(cve, issue, eulerScore); err != nil { - logs.Error("modifiyCve error: ", err.Error(), - "cveNum: ", issue.CveNum, "repo: ", issue.Repo) - continue - } - - if cve.SecurityNoticeNo == "" { - continue - } - - // 修改cvrf文件 - cvrfFilePath, err := modifyCvrfFile(cve, issue, eulerScore, cvrfdir) - if err != nil { - logs.Error("modifyCvrfFile error: ", err.Error(), - "cveNum: ", issue.CveNum, "repo: ", issue.Repo) - continue - } - - modifiedCvrfFiles = append(modifiedCvrfFiles, cvrfFilePath) - } - - if len(modifiedCvrfFiles) > 0 { - content := strings.Join(modifiedCvrfFiles, "\n") - - err := obs.Upload(cvrfdir+modifiedCvrfTxt, []byte(content)) - if err != nil { - logs.Error("upload cvrf file error: ", err.Error(), "cvrfFilePath: ", cvrfdir+modifiedCvrfTxt) - } - } -} - -// 检查issue受影响的分支是否都已经关联PR并且PR都合并 -func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []string) ( - relatedPRNotMerged []string, err error, -) { - endpoint := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests?access_token=%s&repo=%s", - issueTemp.Owner, issueTemp.IssueNum, token, issueTemp.Repo, - ) - req, err := http.NewRequest(http.MethodGet, endpoint, nil) - if err != nil { - return relatedPRNotMerged, err - } - - var prs []gitee.PullRequest - cli := utils.NewHttpClient(defaultClientTimeout) - bytes, _, err := cli.Download(req) - if err != nil { - return relatedPRNotMerged, err - } - - if err := json.Unmarshal(bytes, &prs); err != nil { - return relatedPRNotMerged, err - } - - mergeVersionSets := sets.NewString() - for _, pr := range prs { - if pr.State == gitee.StatusMerged { - mergeVersionSets.Insert(pr.Base.Ref) - } - } - - for _, v := range versions { - if !mergeVersionSets.Has(v) { - relatedPRNotMerged = append(relatedPRNotMerged, v) - } - } - - return relatedPRNotMerged, nil -} - -// ResponseData respresent the response data -type ResponseData struct { - Code int `json:"code"` - Result CveData `json:"result"` - Msg string `json:"msg"` -} - -// CveData respresent the cve data -type CveData struct { - CveDatabase - SecurityNoticeNo string `json:"securityNoticeNo"` - ParserBean *CveParser `json:"parserBean"` - Cvrf *CveCvrf `json:"cvrf"` - PackageList []CveProductPackage `json:"packageList"` -} - -// CveDatabase respresent the cve database -type CveDatabase struct { - Id int64 `json:"id"` - AffectedProduct string `json:"affectedProduct"` - AnnouncementTime string `json:"announcementTime"` - AttackComplexitynvd string `json:"attackComplexityNVD"` - AttackComplexityoe string `json:"attackComplexityOE"` - AttackVectornvd string `json:"attackVectorNVD"` - AttackVectoroe string `json:"attackVectorOE"` - Availabilitynvd string `json:"availabilityNVD"` - Availabilityoe string `json:"availabilityOE"` - Confidentialitynvd string `json:"confidentialityNVD"` - Confidentialityoe string `json:"confidentialityOE"` - CveId string `json:"cveId"` - CvsssCorenvd string `json:"cvsssCoreNVD"` - CvsssCoreoe string `json:"cvsssCoreOE"` - Integritynvd string `json:"integrityNVD"` - Integrityoe string `json:"integrityOE"` - NationalCyberAwarenessSystem string `json:"nationalCyberAwarenessSystem"` - PackageName string `json:"packageName"` - PrivilegesRequirednvd string `json:"privilegesRequiredNVD"` - PrivilegesRequiredoe string `json:"privilegesRequiredOE"` - Scopenvd string `json:"scopeNVD"` - Scopeoe string `json:"scopeOE"` - Status string `json:"status"` - Summary string `json:"summary"` - Type string `json:"type"` - UserInteractionnvd string `json:"userInteractionNVD"` - UserInteractionoe string `json:"userInteractionOE"` - Updateime string `json:"updateTime"` - CreateTime string `json:"createTime"` -} - -// CveParser respresent the cve parser -type CveParser struct { - Id int64 `json:"id"` - Cve string `json:"cve"` - Cvss string `json:"cvss"` - Exception string `json:"exception"` - PackageName string `json:"packageName"` - Score string `json:"score"` - SeverityDetail string `json:"severityDetail"` - Vector string `json:"vector"` - Updateime string `json:"updateTime"` -} - -// CveCvrf respresent the cve cvrf -type CveCvrf struct { - Id int64 `json:"id"` - FileName string `json:"fileName"` - CveId string `json:"cveId"` - Cvrf string `json:"cvrf"` - PackageName string `json:"packageName"` - SecurityNoticeNo string `json:"securityNoticeNo"` - Updateime string `json:"updateTime"` -} - -// CveProductPackage respresent the cve product package -type CveProductPackage struct { - Id int64 `json:"id"` - CveId string `json:"cveId"` - PackageName string `json:"packageName"` - ProductName string `json:"productName"` - Status string `json:"status"` - Reason string `json:"reason"` - SecurityNoticeNo string `json:"securityNoticeNo"` - ReleaseTime string `json:"releaseTime"` - Updateime string `json:"updateTime"` - CreateTime string `json:"createTime"` -} - -// 从cve-sa-backend服务调用接口获取cvedatabase信息 -func getCveByIdAndPackageName(cveId, packageName string) (CveData, error) { - cli := utils.NewHttpClient(defaultClientTimeout) - - url := fmt.Sprintf("%s/cve-security-notice-server/cvedatabase/getByCveIdAndPackageName?cveId=%s&packageName=%s", - endpoint, cveId, packageName) - - request, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - return CveData{}, err - } - - r, _, err := cli.Download(request) - if err != nil { - return CveData{}, err - } - - var resp ResponseData - if err = json.Unmarshal(r, &resp); err != nil { - return CveData{}, err - } - - if resp.Code != 0 { - err = errors.New(resp.Msg) - - return CveData{}, err - } - - return resp.Result, nil -} - -// 从cve-sa-backend服务调用接口更新cvedatabase信息 -func updateCveDatabase(cveDatabase CveDatabase) error { - cli := utils.NewHttpClient(defaultClientTimeout) - - url := fmt.Sprintf("%s/cve-security-notice-server/cvedatabase/updateCveDatabase", endpoint) - - requestBody, err := json.Marshal(cveDatabase) - if err != nil { - return err - } - - request, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(requestBody)) - if err != nil { - return err - } - - request.Header.Set("Content-Type", "application/json") - - _, _, err = cli.Download(request) - - return err -} - -func checkIssueTemp(issueTemp models.IssueTemplate, token string) (string, bool) { - if issueTemp.OpenEulerScore <= 0 { - return "openeuler评分不能为0！", false - } - - score, err := models.QueryIssueScore(issueTemp.CveId) - if err != nil { - logs.Error("query issue score error: ", err.Error()) - return "", false - } - - // 如果score type 为 v3, 则检查 cvss v3 vector 和 score 是否一致 - if score.ScoreType == cvssV3 { - if util.CalculateCVSSV3BaseScore(issueTemp.OpenEulerVector) != issueTemp.OpenEulerScore { - tip := fmt.Sprintf("@%v cvss v3 vector: %v and score: %f are not consistent", - issueTemp.Assignee, issueTemp.OpenEulerVector, issueTemp.OpenEulerScore) - - return tip, false - } - } - - cveCenter := models.VulnCenter{CveId: issueTemp.CveId, CveNum: issueTemp.CveNum} - err = models.GetVulnCenterByCid(&cveCenter, "cve_id", "cve_num") - if err != nil { - logs.Error("get vuln center error: ", err.Error()) - return "", false - } - - // 检查issue的影响性分析是否完成 - if msg, _, ok := taskhandler.CheckIssueAnalysisComplete(&issueTemp, cveCenter.OrganizationID); !ok { - na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" - tip := fmt.Sprintf("@%v %v", issueTemp.Assignee, msg) + na - - return tip, false - } - - // 检查issue的受影响分支关联的PR是否合入 - versions := strings.Split(issueTemp.AffectedVersion, ",") - affectedVersions := make([]string, 0) - for _, v := range versions { - if !strings.Contains(v, "不受影响") { - affectedVersions = append(affectedVersions, v) - } - } - - relatedPRNotMerged, err := checkRelatedPR(issueTemp, token, affectedVersions) - if err != nil { - logs.Error("check related PR error: ", err.Error()) - return "", false - } - - if len(relatedPRNotMerged) > 0 { - tip := fmt.Sprintf("@%s\n"+ - "关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #%s\n"+ - "受影响分支: %s\n"+ - "具体操作参考: %s\n", - issueTemp.Assignee, issueTemp.IssueNum, strings.Join(relatedPRNotMerged, "/"), referenceUrl) - - return tip, false - } - - return "", true -} - -func modifiyCve(cve CveData, issue models.IssueTemplate, eulerScore string) error { - cve.CvsssCoreoe = eulerScore - - // 检查len(vectors) == 8 - vectors := strings.Split(issue.OpenEulerVector, "/") - if len(vectors) != vectorLenth { - return errors.New("invalid openeuler vector") - } - - cve.AttackVectoroe = mapVector[vectors[0]] - cve.AttackComplexityoe = mapVector[vectors[1]] - cve.PrivilegesRequiredoe = mapVector[vectors[2]] - cve.UserInteractionoe = mapVector[vectors[3]] - cve.Scopeoe = mapVector[vectors[4]] - cve.Confidentialityoe = mapVector[vectors[5]] - cve.Integrityoe = mapVector[vectors[6]] - cve.Availabilityoe = mapVector[vectors[7]] - - // 更新cvedatabase - return updateCveDatabase(cve.CveDatabase) -} - -func modifyCvrfFile(cve CveData, issue models.IssueTemplate, eulerScore, cvrfdir string) (string, error) { - obs := obsimpl.Instance() - securityNoticeNo := strings.Split(cve.SecurityNoticeNo, "-") - if len(securityNoticeNo) != saLenth { - return "", errors.New("invalid security notice no") - } - - year := securityNoticeNo[2] - - cvrfFilePath := fmt.Sprintf("%s%s/%s", cvrfdir, year, cve.SecurityNoticeNo) - - content, err := obs.Download(cvrfFilePath) - if err != nil { - return "", err - } - - var cvrf bulletinimpl.Cvrf - err = xml.Unmarshal(content, &cvrf) - if err != nil { - return "", err - } - - for i := range cvrf.Vulnerability { - if cvrf.Vulnerability[i].CVE == issue.CveNum { - cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.BaseScore = eulerScore - cvrf.Vulnerability[i].CVSSScoreSets.ScoreSet.Vector = issue.OpenEulerVector - - break - } - } - - uploadBys, err := xml.MarshalIndent(cvrf, "", " ") - if err != nil { - return "", err - } - - headerBytes := []byte(xml.Header) - headerBytes = append(headerBytes, uploadBys...) - - if err = obs.Upload(cvrfFilePath, headerBytes); err != nil { - return "", err - } - - return cvrfFilePath, nil -} -- Gitee From 5f14a7c640380662689e33522681e666c3aa615d Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Thu, 2 Jan 2025 20:17:36 +0800 Subject: [PATCH 5/7] fix check code --- cve-vulner-manager/task/anomalousissuetask.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/cve-vulner-manager/task/anomalousissuetask.go b/cve-vulner-manager/task/anomalousissuetask.go index 9d211f8..ef046f0 100644 --- a/cve-vulner-manager/task/anomalousissuetask.go +++ b/cve-vulner-manager/task/anomalousissuetask.go @@ -1,13 +1,8 @@ +// Package task represent the task package task import ( "bytes" - "cvevulner/common" - "cvevulner/cve-ddd/infrastructure/bulletinimpl" - "cvevulner/cve-ddd/infrastructure/obsimpl" - "cvevulner/models" - "cvevulner/taskhandler" - "cvevulner/util" "encoding/json" "encoding/xml" "errors" @@ -22,6 +17,13 @@ import ( "github.com/opensourceways/go-gitee/gitee" "github.com/opensourceways/server-common-lib/utils" "k8s.io/apimachinery/pkg/util/sets" + + "cvevulner/common" + "cvevulner/cve-ddd/infrastructure/bulletinimpl" + "cvevulner/cve-ddd/infrastructure/obsimpl" + "cvevulner/models" + "cvevulner/taskhandler" + "cvevulner/util" ) const month = 6 @@ -219,18 +221,19 @@ func checkRelatedPR(issueTemp models.IssueTemplate, token string, versions []str return relatedPRNotMerged, nil } -// ResponseData respresent the response data +// ResponseCveData respresent the response cve data type ResponseCveData struct { Result CveData `json:"result"` CommonData } +// ResponseCveProductPackage respresent the response cve product package type ResponseCveProductPackage struct { Result []CveProductPackage `json:"result"` CommonData } -// ResponseData respresent the response data +// CommonData respresent the common data type CommonData struct { Code int `json:"code"` Msg string `json:"msg"` -- Gitee From cdab87a812a10f39c2648292e06c814015d35f54 Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Tue, 7 Jan 2025 09:53:13 +0800 Subject: [PATCH 6/7] fix --- cve-vulner-manager/task/anomalousissuetask.go | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/cve-vulner-manager/task/anomalousissuetask.go b/cve-vulner-manager/task/anomalousissuetask.go index ef046f0..363d895 100644 --- a/cve-vulner-manager/task/anomalousissuetask.go +++ b/cve-vulner-manager/task/anomalousissuetask.go @@ -26,7 +26,11 @@ import ( "cvevulner/util" ) -const month = 6 +const ( + month = 6 + timeFormat = "2006.01.02" + addNewBranchCfgTime = "2024.11.15" +) // DealAnomalousIssues deal with abnormal issues func DealAnomalousIssues() error { @@ -422,10 +426,19 @@ func checkIssueTemp(issueTemp models.IssueTemplate, token string) (string, error // 检查issue的影响性分析是否完成 if msg, _, ok := taskhandler.CheckIssueAnalysisComplete(&issueTemp, cveCenter.OrganizationID); !ok { - na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" - tip := fmt.Sprintf("@%v %v", issueTemp.Assignee, msg) + na + t, err := time.Parse(timeFormat, addNewBranchCfgTime) + if err != nil { + return "", fmt.Errorf("parse time error: %w", err) + } - return tip, nil + if strings.Contains(msg, "没有分析或未按正确格式填写，涉及分支") { + if issueTemp.CreateTime.After(t) { + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + tip := fmt.Sprintf("@%v %v", issueTemp.Assignee, msg) + na + + return tip, nil + } + } } // 检查issue的受影响分支关联的PR是否合入 @@ -433,7 +446,8 @@ func checkIssueTemp(issueTemp models.IssueTemplate, token string) (string, error affectedVersions := make([]string, 0) for _, v := range versions { if !strings.Contains(v, "不受影响") { - affectedVersions = append(affectedVersions, v) + affectedVersion := strings.Split(v, ":受影响") + affectedVersions = append(affectedVersions, affectedVersion[0]) } } -- Gitee From 2831cf9cc9535319b84641a89501bdcfb1888090 Mon Sep 17 00:00:00 2001 From: Coopermassaki <1277145053@qq.com> Date: Tue, 7 Jan 2025 10:38:53 +0800 Subject: [PATCH 7/7] fix --- cve-vulner-manager/task/anomalousissuetask.go | 92 ++++++++++++------- 1 file changed, 57 insertions(+), 35 deletions(-) diff --git a/cve-vulner-manager/task/anomalousissuetask.go b/cve-vulner-manager/task/anomalousissuetask.go index 363d895..70501b5 100644 --- a/cve-vulner-manager/task/anomalousissuetask.go +++ b/cve-vulner-manager/task/anomalousissuetask.go @@ -425,48 +425,22 @@ func checkIssueTemp(issueTemp models.IssueTemplate, token string) (string, error } // 检查issue的影响性分析是否完成 - if msg, _, ok := taskhandler.CheckIssueAnalysisComplete(&issueTemp, cveCenter.OrganizationID); !ok { - t, err := time.Parse(timeFormat, addNewBranchCfgTime) - if err != nil { - return "", fmt.Errorf("parse time error: %w", err) - } - - if strings.Contains(msg, "没有分析或未按正确格式填写，涉及分支") { - if issueTemp.CreateTime.After(t) { - na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" - tip := fmt.Sprintf("@%v %v", issueTemp.Assignee, msg) + na - - return tip, nil - } - } + tip, err := checkIssueAnalysis(&issueTemp, cveCenter.OrganizationID) + if err != nil { + return "", fmt.Errorf("check issue analysis error: %w", err) } - // 检查issue的受影响分支关联的PR是否合入 - versions := strings.Split(issueTemp.AffectedVersion, ",") - affectedVersions := make([]string, 0) - for _, v := range versions { - if !strings.Contains(v, "不受影响") { - affectedVersion := strings.Split(v, ":受影响") - affectedVersions = append(affectedVersions, affectedVersion[0]) - } + if tip != "" { + return tip, nil } - relatedPRNotMerged, err := checkRelatedPR(issueTemp, token, affectedVersions) + // 检查issue的受影响分支关联的PR是否合入 + tip, err = checkRelatedPRAndGenerateTip(issueTemp, token) if err != nil { - return "", fmt.Errorf("check related pr error: %w", err) - } - - if len(relatedPRNotMerged) > 0 { - tip := fmt.Sprintf("@%s\n"+ - "关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #%s\n"+ - "受影响分支: %s\n"+ - "具体操作参考: %s\n", - issueTemp.Assignee, issueTemp.IssueNum, strings.Join(relatedPRNotMerged, "/"), referenceUrl) - - return tip, nil + return "", err } - return "", nil + return tip, nil } func modifiyCve(cve CveData, issue models.IssueTemplate, eulerScore string) error { @@ -556,3 +530,51 @@ func modifyCvrfFile(cve CveData, issue models.IssueTemplate, eulerScore, cvrfdir return cvrfFiles.List(), nil } + +func checkIssueAnalysis(issueTemp *models.IssueTemplate, orgId int8) (string, error) { + if msg, _, ok := taskhandler.CheckIssueAnalysisComplete(issueTemp, orgId); !ok { + t, err := time.Parse(timeFormat, addNewBranchCfgTime) + if err != nil { + return "", fmt.Errorf("parse time error: %w", err) + } + + if strings.Contains(msg, "没有分析或未按正确格式填写，涉及分支") { + if issueTemp.CreateTime.After(t) { + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + tip := fmt.Sprintf("@%v %v", issueTemp.Assignee, msg) + na + + return tip, nil + } + } + } + + return "", nil +} + +func checkRelatedPRAndGenerateTip(issueTemp models.IssueTemplate, token string) (string, error) { + versions := strings.Split(issueTemp.AffectedVersion, ",") + affectedVersions := make([]string, 0) + for _, v := range versions { + if !strings.Contains(v, "不受影响") { + affectedVersion := strings.Split(v, ":受影响") + affectedVersions = append(affectedVersions, affectedVersion[0]) + } + } + + relatedPRNotMerged, err := checkRelatedPR(issueTemp, token, affectedVersions) + if err != nil { + return "", fmt.Errorf("check related pr error: %w", err) + } + + if len(relatedPRNotMerged) > 0 { + tip := fmt.Sprintf("@%s\n"+ + "关闭issue前,需要将受影响的分支在合并pr时关联上当前issue编号: #%s\n"+ + "受影响分支: %s\n"+ + "具体操作参考: %s\n", + issueTemp.Assignee, issueTemp.IssueNum, strings.Join(relatedPRNotMerged, "/"), referenceUrl) + + return tip, nil + } + + return "", nil +} -- Gitee