From 59edcfce925accea1595594b689d89d0d4f2184b Mon Sep 17 00:00:00 2001 From: zjm <1076115376@qq.com> Date: Thu, 1 Jun 2023 16:16:47 +0800 Subject: [PATCH 1/2] generate hotpatch --- cve-vulner-manager/controllers/file.go | 3 +- cve-vulner-manager/taskhandler/cve.go | 4 +- cve-vulner-manager/taskhandler/cvrf.go | 170 +++++++++++++++++- cve-vulner-manager/taskhandler/excel.go | 58 +++--- cve-vulner-manager/taskhandler/patch_issue.go | 155 ++++++++++++++++ 5 files changed, 358 insertions(+), 32 deletions(-) create mode 100644 cve-vulner-manager/taskhandler/patch_issue.go diff --git a/cve-vulner-manager/controllers/file.go b/cve-vulner-manager/controllers/file.go index e05da7f..94aa251 100644 --- a/cve-vulner-manager/controllers/file.go +++ b/cve-vulner-manager/controllers/file.go @@ -175,6 +175,7 @@ func (f *FileController) TriggerCveData() { unaffectcvrf.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" unaffectcvrf.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" du := beego.AppConfig.DefaultString("excel::v_pack_20_03_url", "") + issues, _ := taskhandler.TestPatchIssues() csvPathList := strings.Split(du, ";") if len(csvPathList) > 0 { for _, csvP := range csvPathList { @@ -190,7 +191,7 @@ func (f *FileController) TriggerCveData() { fileCode := common.EncryptMd5(en) wgCrvf.Add(1) updateInfoxml := taskhandler.GenerateExcelTrigger(&wgCrvf, en, startTime, fileCode, affectBranch, - csvDownPath, dir, cvrfFileList, componentMap, cvrfFileMap, cves) + csvDownPath, dir, cvrfFileList, componentMap, cvrfFileMap, cves, issues) updateInfoSlice = append(updateInfoSlice, updateInfoxml) } } diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index afaf11c..5e988cb 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -2630,7 +2630,7 @@ func GenerateExcelTask() error { //GenerateExcelTrigger generate cve&security notice excel file by pr merge and influence package release. func GenerateExcelTrigger(wgCrvf *sync.WaitGroup, fileName, startTime, fileCode, affectBranch, csvDownPath, dir string, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, cves []string) UpdateInfoXml { + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, cves []string, issues []patchIssue) UpdateInfoXml { defer wgCrvf.Done() logs.Info(affectBranch, ", GenerateExcelTrigger start...") var updateInfoXml UpdateInfoXml @@ -2658,7 +2658,7 @@ func GenerateExcelTrigger(wgCrvf *sync.WaitGroup, fileName, startTime, fileCode, err = GenerateCveExcelByTrigger(affectBranch, fileName, snPrefix, startTime, snSuffix, true, pkgList, cvrfFileList, componentMap, cvfrFileMap, &cvexml, &dpdates, - securityNotice, packRpmx, cves) + securityNotice, packRpmx, cves, issues) if err != nil { logs.Error(err) } diff --git a/cve-vulner-manager/taskhandler/cvrf.go b/cve-vulner-manager/taskhandler/cvrf.go index 04e65a5..2c27c7c 100644 --- a/cve-vulner-manager/taskhandler/cvrf.go +++ b/cve-vulner-manager/taskhandler/cvrf.go @@ -22,6 +22,21 @@ import ( "github.com/astaxie/beego/logs" ) +const ( + aarch64 = "aarch64" + aarch64Suffix = ".aarch64." + noarch = "noarch" + noarchSuffix = ".noarch." + x8664 = "x86_64" + x8664suffix = ".x86_64." + src = "src" + srcSuffix = ".src." + productName = "Product Name" + packageArch = "Package Arch" +) + +var archs = []string{aarch64, noarch, x8664, src} + type CvrfSa struct { XMLName xml.Name `xml:"cvrfdoc,omitempty"` Xmlns string `xml:"xmlns,attr"` @@ -33,6 +48,7 @@ type CvrfSa struct { DocumentNotes *DocumentNotes `xml:"DocumentNotes,omitempty"` DocumentReferences *DocumentReferences `xml:"DocumentReferences,omitempty"` ProductTree *ProductTree `xml:"ProductTree,omitempty"` + HotPatchTree *HotPatchTree `xml:"HotPatchTree,omitempty"` Vulnerability []Vulnerability `xml:"Vulnerability,omitempty"` } @@ -119,6 +135,12 @@ type ProductTree struct { OpenEulerBranch []OpenEulerBranch `xml:"Branch,omitempty"` } +type HotPatchTree struct { + XMLName xml.Name `xml:"HotPatchTree,omitempty"` + Xmlns string `xml:"xmlns,attr"` + OpenEulerBranch []OpenEulerBranch `xml:"Branch,omitempty"` +} + type OpenEulerBranch struct { XMLName xml.Name `xml:"Branch,omitempty"` Type string `xml:"Type,attr"` @@ -130,6 +152,7 @@ type FullProductName struct { XMLName xml.Name `xml:"FullProductName,omitempty"` ProductId string `xml:"ProductID,attr"` Cpe string `xml:"CPE,attr"` + Cve string `xml:"CVE,attr,omitempty"` FullProductName string `xml:",innerxml"` } @@ -869,6 +892,23 @@ func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, } } +func isExistHotPatch(v []FullProductName, pkg, cpe, cve string) bool { + for _, fp := range v { + if fp.FullProductName == pkg && fp.Cpe == cpe && fp.Cve == cve { + return true + } + } + + return false +} + +func genCpe(branch string) string { + affectBranchListx := strings.Split(branch, "-") + return fmt.Sprintf( + "cpe:/a:%s:%s:%s", + affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) +} + func BuildProductTree(cvrfsa *CvrfSa, v models.ExcelExport, pkg []models.Package, affectBranch string, branchFlag int) { if cvrfsa.ProductTree != nil && len(cvrfsa.ProductTree.OpenEulerBranch) > 0 && branchFlag == 1 { @@ -1076,6 +1116,117 @@ func BuildProductTree(cvrfsa *CvrfSa, v models.ExcelExport, pkg []models.Package } } +func BuildHotPatchTree(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, branchFlag int, issues []patchIssue) { + var patch patchIssue + for i := range issues { + if issues[i].FindCve(v.CveNum, v.InfluenceComponent, affectBranch) { + patch = issues[i] + break + } + } + if patch.Type == "" { + return + } + cpe := genCpe(affectBranch) + if cvrfsa.HotPatchTree != nil && len(cvrfsa.HotPatchTree.OpenEulerBranch) > 0 { + for i := range cvrfsa.HotPatchTree.OpenEulerBranch { + opBranch := &cvrfsa.HotPatchTree.OpenEulerBranch[i] + if opBranch.Type == productName && strings.Contains(affectBranch, opBranch.Name) { + fpFlag := false + for _, fp := range opBranch.FullProductName { + if fp.FullProductName == affectBranch { + fpFlag = true + break + } + } + if !fpFlag { + opBranch.FullProductName = append( + opBranch.FullProductName, genFullProductName("", cpe, affectBranch, affectBranch), + ) + } + } + if opBranch.Type == packageArch { + for _, s := range patch.PatchUrl { + pro := genFullProductName(patch.CveNum, cpe, checkPatchUrl(s), genRpm(s)) + for _, arch := range archs { + if strings.HasSuffix(s, arch+".rpm") { + if findHotTree(cvrfsa, arch) == -1 { + cvrfsa.HotPatchTree.OpenEulerBranch = append( + cvrfsa.HotPatchTree.OpenEulerBranch, + genOpenEulerBranch(packageArch, arch, []FullProductName{pro}), + ) + break + } + if !isExistHotPatch(opBranch.FullProductName, s, cpe, patch.CveNum) && opBranch.Name == arch { + opBranch.FullProductName = append(opBranch.FullProductName, pro) + } + } + } + } + } + } + } else { + cvrfsa.HotPatchTree = &HotPatchTree{ + Xmlns: "http://www.icasi.org/CVRF/schema/prod/1.1", + OpenEulerBranch: []OpenEulerBranch{ + genOpenEulerBranch( + productName, strings.Split(affectBranch, "-")[0], + []FullProductName{genFullProductName("", cpe, affectBranch, affectBranch)}, + ), + }, + } + + for _, s := range patch.PatchUrl { + for _, arch := range archs { + if strings.HasSuffix(s, arch+".rpm") { + pro := genFullProductName(patch.CveNum, cpe, checkPatchUrl(s), genRpm(s)) + idx := findHotTree(cvrfsa, arch) + if idx == -1 { + cvrfsa.HotPatchTree.OpenEulerBranch = append( + cvrfsa.HotPatchTree.OpenEulerBranch, + genOpenEulerBranch(packageArch, arch, []FullProductName{pro}), + ) + } else { + branch := &cvrfsa.HotPatchTree.OpenEulerBranch[idx] + if !isExistHotPatch(branch.FullProductName, s, cpe, patch.CveNum) { + branch.FullProductName = append(branch.FullProductName, pro) + } + } + } + } + } + } +} + +func genFullProductName(cve, cpe, productID, proName string) FullProductName { + return FullProductName{ + ProductId: productID, + Cpe: cpe, + Cve: cve, + FullProductName: proName, + } +} + +func genOpenEulerBranch(typ, name string, full []FullProductName) OpenEulerBranch { + return OpenEulerBranch{Type: typ, Name: name, FullProductName: full} +} + +func findHotTree(sa *CvrfSa, arch string) int { + index := -1 + + if sa.HotPatchTree == nil { + return index + } + + for i, v := range sa.HotPatchTree.OpenEulerBranch { + if v.Name == arch { + return i + } + } + + return index +} + func checkRpm(rpm string) (s string, ok bool) { index := strings.LastIndex(rpm, ".oe1.") if index > 0 { @@ -1093,6 +1244,22 @@ func checkRpm(rpm string) (s string, ok bool) { return } +func checkPatchUrl(u string) (s string) { + if list := strings.Split(u, "/"); len(list) > 0 { + s = strings.Split(list[len(list)-1], ".")[0] + } + + return +} + +func genRpm(u string) (rpm string) { + if list := strings.Split(u, "/"); len(list) > 0 { + rpm = list[len(list)-1] + } + + return +} + func BuildVulnerability(vlLenth int, v models.ExcelExport, componentMap map[string]ComponentInfo, cpe string, branchFlag int) []Vulnerability { vulnerabilitySlice := make([]Vulnerability, 0) @@ -1219,7 +1386,7 @@ func BuildVulnerabilitySet(cvrfsa *CvrfSa, v models.ExcelExport, func BuildCvrfXml(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, - pkg []models.Package, branchFlag int) { + pkg []models.Package, branchFlag int, issues []patchIssue) { curDate := common.GetCurDate() branchList := BranchExist(affectBranch, cvrfFileList) cvrfsa.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" @@ -1227,6 +1394,7 @@ func BuildCvrfXml(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, BuildDocumentTitle(cvrfsa, v, affectBranch, cvrfFileList, componentMap, curDate, branchList, branchFlag) BuildDocumentRef(cvrfsa, v, componentMap, branchFlag, affectBranch) BuildProductTree(cvrfsa, v, pkg, affectBranch, branchFlag) + BuildHotPatchTree(cvrfsa, v, affectBranch, branchFlag, issues) BuildVulnerabilitySet(cvrfsa, v, affectBranch, componentMap, branchFlag) } diff --git a/cve-vulner-manager/taskhandler/excel.go b/cve-vulner-manager/taskhandler/excel.go index f4263a0..9271996 100644 --- a/cve-vulner-manager/taskhandler/excel.go +++ b/cve-vulner-manager/taskhandler/excel.go @@ -92,7 +92,7 @@ func GenerateCveExcelByTrigger(affectBranch, excelName, snPrefix, startTime stri forceRewrite bool, pkgList []models.ExcelPackage, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, cvexml *[]CveXml, dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, - packRpmx map[string][]PackRpm, cves []string) (err error) { + packRpmx map[string][]PackRpm, cves []string, issues []patchIssue) (err error) { if len(pkgList) == 0 { return errors.New("No data to export! ") } @@ -113,7 +113,7 @@ func GenerateCveExcelByTrigger(affectBranch, excelName, snPrefix, startTime stri } ec.FillContentTrigger(pkgList, startTime, affectBranch, cvrfFileList, componentMap, cvfrFileMap, cvexml, dpdates, - securityNotice, packRpmx, cves) + securityNotice, packRpmx, cves, issues) return ec.Save(mode) } @@ -400,7 +400,7 @@ func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, affectBranch string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, cvexml *[]CveXml, dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, - packRpmx map[string][]PackRpm, cves []string) { + packRpmx map[string][]PackRpm, cves []string, issues []patchIssue) { pl := len(pkgList) pageSize := 10 pc := pl / 10 @@ -422,8 +422,9 @@ func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, for i := 0; i < pc; i++ { wgTrigger.Add(1) go ec.handleGiteData(cd, affectBranch, cvexml, dpdates, - securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap) + securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap, issues) } + wgTrigger.Wait() } @@ -462,7 +463,7 @@ func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { if lz > 0 { for _, v := range list { if v.Num == 1 { - ec.setContentRow(v, "", cvrfFileList, componentMap, cvfrFileMap) + ec.setContentRow(v, "", cvrfFileList, componentMap, cvfrFileMap, nil) } else if v.Num > 1 { //1.Obtain issue_tpl according to cve_num, if all issue_status == 2 0r issue_status == 6, then data can be exported list, err := models.GetIssueTplByCveNum(v.CveNum) @@ -500,7 +501,7 @@ func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { ep.AffectProduct = ep.AffectProduct + "\n" + ex.AffectProduct } } - ec.setContentRow(ep, "", cvrfFileList, componentMap, cvfrFileMap) + ec.setContentRow(ep, "", cvrfFileList, componentMap, cvfrFileMap, nil) } } } @@ -512,7 +513,7 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBranch string, cvexml *[]CveXml, dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) (err error) { + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, issues []patchIssue) (err error) { lz := len(list) if lz > 0 { for _, v := range list { @@ -526,7 +527,7 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, continue } fillLock.Lock() - ec.setContentRow(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap) + ec.setContentRow(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap, issues) fillLock.Unlock() fillLock.Lock() BuildXml(cvexml, &v, securityNotice, packRpmx, affectBranch) @@ -850,10 +851,10 @@ func SaveCvrfName(cvrfFileList map[string][]string, cvrffileName, mapKey string) } } -func BranchCvrfData(v models.ExcelExport, - affectBranch, cvrffileName string, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, - cvfrFileMap map[string]CvrfSa, pkg []models.Package, branchFlag int) { +func BranchCvrfData( + v models.ExcelExport, affectBranch, cvrffileName string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, + cvfrFileMap map[string]CvrfSa, pkg []models.Package, branchFlag int, issues []patchIssue, +) { branchCvrfSaStruct, bCvrfSaOk := cvfrFileMap[cvrffileName] if !bCvrfSaOk { // Query whether there is data in the database @@ -878,28 +879,29 @@ func BranchCvrfData(v models.ExcelExport, componentMap[v.InfluenceComponent] = componentInfo } } - BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) + BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag, issues) cvfrFileMap[cvrffileName] = cvrfsa } else { - BuildCvrfXml(&branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) + BuildCvrfXml(&branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag, issues) cvfrFileMap[cvrffileName] = branchCvrfSaStruct } + SaveCvrfName(cvrfFileList, cvrffileName, CVRFFKEY) } func procCvrfData(v models.ExcelExport, affectBranch string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, - cvfrFileMap map[string]CvrfSa, pkg []models.Package) { + cvfrFileMap map[string]CvrfSa, pkg []models.Package, issues []patchIssue) { totalCvrffileName := componentMap[v.InfluenceComponent].CvrfFileName branchCvrffileName := componentMap[v.InfluenceComponent].CveNumMap[affectBranch].CvrfFileName - BranchCvrfData(v, affectBranch, branchCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 2) - BranchCvrfData(v, affectBranch, totalCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 1) + BranchCvrfData(v, affectBranch, branchCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 2, issues) + BranchCvrfData(v, affectBranch, totalCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 1, issues) } func (ec *CveExcel) setContentRow(v models.ExcelExport, affectBranch string, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, issues []patchIssue) { pkg, pkgErr := models.GetCvePackageList(v.SecID, affectBranch) if pkgErr != nil { logs.Error(pkgErr) @@ -910,7 +912,7 @@ func (ec *CveExcel) setContentRow(v models.ExcelExport, pkgStr := getPkgStr(pkg) repFlag := StoreComponentInfo(componentMap, v, affectBranch) if !repFlag { - procCvrfData(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap, pkg) + procCvrfData(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap, pkg, issues) sn := []interface{}{v.OpenEulerSANum, v.CveNum, v.Introduction, v.Summary, v.Theme, v.Description, v.InfluenceComponent, v.AffectProduct, pkgStr, v.ReferenceLink, v.PublicDate} @@ -1127,7 +1129,7 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I // Query the current issues that need to be dealt with issueTemp, err := models.GetIssueNumber(v.Repo, cves) if err != nil || issueTemp == nil { - logs.Info("No need to deal with ", v.Repo) + //logs.Info("No need to deal with ", v.Repo) continue } for _, isTemp := range issueTemp { @@ -1154,10 +1156,11 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, cvexml *[]CveXml, dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { + componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, issues []patchIssue) { defer wgTrigger.Done() data := <-c var pkgList []string + patchBranchs := patchBranchIssue(issues, affectBranch) for _, v := range data { //parse package string to list pkgList = strings.Split(v.IssuePkg, " ") @@ -1166,7 +1169,7 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, continue } affectIssueProc(v, affectBranch, cvexml, dpdates, securityNotice, packRpmx, - cvrfFileList, componentMap, cvfrFileMap, pkgList, ec) + cvrfFileList, componentMap, cvfrFileMap, pkgList, ec, patchBranchs) } } @@ -1271,10 +1274,10 @@ func affectIssueProc(v IssueAndPkg, affectBranch string, cvexml *[]CveXml, dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, - pkgList []string, ec *CveExcel) { + pkgList []string, ec *CveExcel, issues []patchIssue) { for _, iv := range v.IssueMap { - tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo, IssueId: v.IssueId} - err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo", "issue_id") + tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo, IssueId: v.IssueId, CveNum: iv.CveNumber} + err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo", "issue_id", "CveNum") if err != nil { logs.Error("GetIssueTemplateByColName, ----", err, iv.Number, iv.Repo, v.IssueId) continue @@ -1303,13 +1306,12 @@ func affectIssueProc(v IssueAndPkg, affectBranch string, cvexml *[]CveXml, continue } } - err = models.ReplacePackageByCveId(pkgList, tpl.CveId, affectBranch) - if err != nil { + if err = models.ReplacePackageByCveId(pkgList, tpl.CveId, affectBranch); err != nil { logs.Error("ReplacePackageByCveId, err: ", err, "tpl.CveId: ", tpl.CveId) continue } err = ec.handleWriteContentSync(el, affectBranch, cvexml, dpdates, - securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap) + securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap, issues) if err != nil { logs.Error("handleWriteContentSync, err: ", err) } diff --git a/cve-vulner-manager/taskhandler/patch_issue.go b/cve-vulner-manager/taskhandler/patch_issue.go new file mode 100644 index 0000000..875bc0d --- /dev/null +++ b/cve-vulner-manager/taskhandler/patch_issue.go @@ -0,0 +1,155 @@ +package taskhandler + +import ( + "encoding/json" + "fmt" + "net/http" + "os" + "regexp" + "strings" + + "cvevulner/util" +) + +const testIssue = "https://gitee.com/api/v5/repos/qinsheng99/cve-manager/issues?access_token=%s&state=closed&labels=%s&sort=created&direction=desc&page=%d&per_page=20" + +var ( + RegexpPackage = regexp.MustCompile(``) + RegexpCve = regexp.MustCompile(`(?s:(.*?))`) + RegexpPatch = regexp.MustCompile(`热补丁路径[:：](?s:(.*?))$`) + RegexpMeta = regexp.MustCompile(`热补丁元数据[:：](?s:(.*?))热补丁路径[:：]`) + RegexpType = regexp.MustCompile(`问题类别[:：](?s:(.*?))热补丁元数据[:：]`) +) + +func TestPatchIssues() ([]patchIssue, error) { + var page = 1 + var patch []Issue + for { + resp, err := http.Get(fmt.Sprintf(testIssue, os.Getenv("GITEE_TOKEN"), "hotpatch", page)) + if err != nil { + return nil, err + } + + var res []Issue + + err = json.NewDecoder(resp.Body).Decode(&res) + if err != nil { + return nil, err + } + + if len(res) == 0 { + break + } + + patch = append(patch, res...) + page++ + } + + if len(patch) == 0 { + return nil, nil + } + + var patchs []patchIssue + for i := range patch { + if pat, err := toPatchIssue(patch[i].Body); err == nil { + patchs = append(patchs, pat) + } + } + + return patchs, nil +} + +type Issue struct { + Body string `json:"body"` + Repository struct { + Namespace struct { + Path string `json:"path"` + } `json:"namespace"` + } `json:"repository"` +} + +func toPatchIssue(body string) (v patchIssue, err error) { + t := RegexpType.FindAllStringSubmatch(body, -1) + if len(t) > 0 && len(t[0]) > 1 { + v.Type = strings.TrimSpace(t[0][1]) + } + var bys []byte + meta := RegexpMeta.FindAllStringSubmatch(body, -1) + if len(meta) > 0 && len(meta[0]) > 1 { + bys, err = util.HTTPGetCom(strings.TrimSpace(meta[0][1])) + if err != nil { + return + } + + if p := RegexpPackage.FindAllStringSubmatch(string(bys), -1); len(p) > 0 && len(p[0]) > 1 { + v.Component = p[0][1] + } + + if i := strings.Index(meta[0][1], "openEuler-"); i >= 0 { + v.Branch = strings.Split(meta[0][1][i:], "/")[0] + } + } + + p := RegexpPatch.FindAllStringSubmatch(body, -1) + if len(p) > 0 && len(p[0]) > 1 { + v.PatchUrl = strings.Split(strings.TrimSpace(p[0][1]), "\n") + } + + for i := range v.PatchUrl { + if strings.Contains(v.PatchUrl[i], ".xml") { + bys, err = util.HTTPGetCom(strings.TrimSpace(v.PatchUrl[i])) + if err != nil { + return + } + + if cve := RegexpCve.FindAllStringSubmatch(string(bys), -1); len(cve) > 0 && len(cve[0]) > 1 { + v.CveNum = cve[0][1] + } + } + } + + return +} + +type patchIssue struct { + Type string + CveNum string + Branch string + Component string + PatchUrl []string + flag bool +} + +func patchBranchIssue(p []patchIssue, branch string) (v []patchIssue) { + for i := range p { + if p[i].FindBranch(branch) { + v = append(v, p[i]) + } + } + + return +} + +func (p *patchIssue) FindCve(cvenum, component, branch string) bool { + if p.CveNum == cvenum && p.Component == component && p.Branch == branch { + return true + } + + return false +} + +func (p *patchIssue) setTrue() { + p.flag = true +} + +func (p *patchIssue) isUsed() bool { + return p.flag +} + +func (p *patchIssue) FindBranch(branch string) bool { + if p.Branch == branch { + return true + } + + return false +} -- Gitee From 4c668094c458f6a646d6867501760d7066232f66 Mon Sep 17 00:00:00 2001 From: zjm <1076115376@qq.com> Date: Mon, 19 Jun 2023 10:45:29 +0800 Subject: [PATCH 2/2] update regexp --- cve-vulner-manager/Dockerfile | 2 +- cve-vulner-manager/taskhandler/patch_issue.go | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cve-vulner-manager/Dockerfile b/cve-vulner-manager/Dockerfile index 6508e47..0a73d0e 100644 --- a/cve-vulner-manager/Dockerfile +++ b/cve-vulner-manager/Dockerfile @@ -7,7 +7,7 @@ COPY . /go/src/gitee.com/openeuler/cve-manager RUN cd /go/src/gitee.com/openeuler/cve-manager && CGO_ENABLED=1 go build -v -o ./cve-manager main.go # copy binary config and utils -FROM openeuler/openeuler:21.03 +FROM openeuler/openeuler:22.03 RUN mkdir -p /opt/app/conf/ COPY ./conf/product_app.conf /opt/app/conf/app.conf # overwrite config yaml diff --git a/cve-vulner-manager/taskhandler/patch_issue.go b/cve-vulner-manager/taskhandler/patch_issue.go index 875bc0d..a716740 100644 --- a/cve-vulner-manager/taskhandler/patch_issue.go +++ b/cve-vulner-manager/taskhandler/patch_issue.go @@ -16,9 +16,10 @@ const testIssue = "https://gitee.com/api/v5/repos/qinsheng99/cve-manager/issues? var ( RegexpPackage = regexp.MustCompile(``) RegexpCve = regexp.MustCompile(`(?s:(.*?))`) - RegexpPatch = regexp.MustCompile(`热补丁路径[:：](?s:(.*?))$`) + RegexpPatch = regexp.MustCompile(`热补丁路径[:：](?s:(.*?))热补丁信息[:：]`) RegexpMeta = regexp.MustCompile(`热补丁元数据[:：](?s:(.*?))热补丁路径[:：]`) RegexpType = regexp.MustCompile(`问题类别[:：](?s:(.*?))热补丁元数据[:：]`) + RegexpMessage = regexp.MustCompile(`热补丁信息[:：](?s:(.*?))$`) ) func TestPatchIssues() ([]patchIssue, error) { @@ -76,7 +77,7 @@ func toPatchIssue(body string) (v patchIssue, err error) { var bys []byte meta := RegexpMeta.FindAllStringSubmatch(body, -1) if len(meta) > 0 && len(meta[0]) > 1 { - bys, err = util.HTTPGetCom(strings.TrimSpace(meta[0][1])) + bys, err = util.HTTPGetCom(strings.Replace(strings.TrimSpace(meta[0][1]), "/blob/", "/raw/", -1)) if err != nil { return } @@ -95,15 +96,17 @@ func toPatchIssue(body string) (v patchIssue, err error) { v.PatchUrl = strings.Split(strings.TrimSpace(p[0][1]), "\n") } - for i := range v.PatchUrl { - if strings.Contains(v.PatchUrl[i], ".xml") { - bys, err = util.HTTPGetCom(strings.TrimSpace(v.PatchUrl[i])) + m := RegexpMessage.FindAllStringSubmatch(body, -1) + if len(m) > 0 && len(m[0]) > 1 { + for _, s := range strings.Split(strings.TrimSpace(m[0][1]), "\n") { + bys, err = util.HTTPGetCom(strings.TrimSpace(s)) if err != nil { - return + continue } if cve := RegexpCve.FindAllStringSubmatch(string(bys), -1); len(cve) > 0 && len(cve[0]) > 1 { v.CveNum = cve[0][1] + break } } } -- Gitee