diff --git a/.gitignore b/.gitignore index cc9cc23448ee55b7fcac6cdde06ec7d266fdd5cc..2c9068925be7612732623087daeae7a03e8a361e 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ __pycache__ .gitee/ logs/ lastupdate.tmp +*.log diff --git a/cve-vulner-manager/common/common.go b/cve-vulner-manager/common/common.go index 850cb4fa2887a88e8fffd2ec48573befcd1b130d..910e636e893a7b1d53dafcafb00a7654dca18f24 100644 --- a/cve-vulner-manager/common/common.go +++ b/cve-vulner-manager/common/common.go @@ -329,32 +329,39 @@ type CveFixSuggest struct { } type CveOriginData struct { - Ids string `json:"ids"` - CveNum string `json:"cveNum"` - UpdateType string `json:"updateType"` - CvePackName []string `json:"cvePackName"` - PackName []string `json:"packName"` - Description CveDescription `json:"description"` - Title string `json:"title"` - AffectProduct []string `json:"affectProduct"` - Configurations CveConfigurations `json:"configurations"` - CnnvdID string `json:"cnnvdID"` - CnvdID string `json:"cnvdID"` - PublishedDate string `json:"publishedDate"` - GetTime string `json:"getTime"` - EndGetTime string `json:"endGetTime"` - Impact CveImpact `json:"impact"` - VulStatus string `json:"vulStatus"` - Poc CvePoc `json:"poc"` - Event []CveEvent `json:"event"` - ReferenceData []CveReferenceData `json:"referenceData"` - VulType []CveVulType `json:"vulType"` - FixSuggest CveFixSuggest `json:"fixSuggest"` - Version string `json:"version"` - Credibility int `json:"credibility"` - Patch []CveOriginPatch `json:"patch"` - PackageUrl CvePackageUrl `json:"package_url"` - AffectUpdate bool `json:"affectUpdate"` + Ids string `json:"ids"` + CveNum string `json:"cveNum"` + UpdateType string `json:"updateType"` + CvePackName []string `json:"cvePackName"` + PackName []string `json:"packName"` + PackageTime []PackageTime `json:"affectProducts"` + Description CveDescription `json:"description"` + Title string `json:"title"` + AffectProduct []string `json:"affectProduct"` + Configurations CveConfigurations `json:"configurations"` + CnnvdID string `json:"cnnvdID"` + CnvdID string `json:"cnvdID"` + PublishedDate string `json:"publishedDate"` + GetTime string `json:"getTime"` + EndGetTime string `json:"endGetTime"` + Impact map[string]CveImpact `json:"impact"` + VulStatus string `json:"vulStatus"` + Poc CvePoc `json:"poc"` + Event []CveEvent `json:"event"` + ReferenceData []CveReferenceData `json:"referenceData"` + VulType []CveVulType `json:"vulType"` + FixSuggest CveFixSuggest `json:"fixSuggest"` + Version string `json:"version"` + Credibility int `json:"credibility"` + Patch []CveOriginPatch `json:"patch"` + PackageUrl CvePackageUrl `json:"package_url"` + AffectUpdate bool `json:"affectUpdate"` +} + +// PackageTime The time when A was first add +type PackageTime struct { + Name string `json:"name"` + AddTime string `json:"addtime"` } type CvePackageUrl struct { diff --git a/cve-vulner-manager/controllers/cve.go b/cve-vulner-manager/controllers/cve.go index 2c3bdc282d5e81c6d335be8402dc5d82963b7dc7..9b205c8b7eaf03ab11f070c841bbf4bb4003a932 100644 --- a/cve-vulner-manager/controllers/cve.go +++ b/cve-vulner-manager/controllers/cve.go @@ -142,6 +142,7 @@ type CveAllIssueoData struct { IsIgnore int8 `json:"is_ignore"` FirstPatchTime string `json:"first_patch_time"` AffectUpdate int8 `json:"affect_update"` + PackageTime string `json:"package_time"` } // @Title Obtain all cve data and provide operation kanban for use @@ -218,6 +219,7 @@ func (u *CveAllIssueController) Get() { IsIgnore: issues.IsIgnore, FirstPatchTime: models.QueryFirstPatchTime(issues.CveNum), AffectUpdate: issues.AffectUpdate, + PackageTime: issues.PackageTime, } } resp["body"] = cid diff --git a/cve-vulner-manager/controllers/file.go b/cve-vulner-manager/controllers/file.go index 88e155353a2aa6a029ce4abc429ce09f4aaad803..2f80684ad81db24493932a936e2f91776b01aac5 100644 --- a/cve-vulner-manager/controllers/file.go +++ b/cve-vulner-manager/controllers/file.go @@ -1,26 +1,14 @@ package controllers import ( - "encoding/xml" "fmt" - "io/ioutil" - "math/rand" - "os" "path/filepath" - "regexp" - "strconv" - "strings" - "sync" - "time" "cvevulner/common" - "cvevulner/cve-ddd/infrastructure/obsimpl" "cvevulner/models" - "cvevulner/taskhandler" "cvevulner/util" "github.com/astaxie/beego" - "github.com/astaxie/beego/logs" ) // FileController file operation routing processing @@ -79,593 +67,3 @@ func (f *FileController) DownLoadExcelByFileCode() { } f.Ctx.Output.Download(fp, er.FileName) } - -// TriggerCveData touch off generate cve data excel and get cve package -// @router /triggerCveData [get] -func (f *FileController) TriggerCveData() { - // 停止维护 - f.Ctx.WriteString("Deprecated api") - return - // Limit on the number of triggers - nameStr, limitCount := LimitTriggerSa() - if limitCount != 0 { - cha := fmt.Sprintf("The number of requests is too frequent, please try again later, " + - "there is currently a task being processed") - f.Ctx.WriteString(cha) - UpdateLimitTriggerSa(nameStr, 2) - return - } - startTime := f.GetString("startTime") - if startTime == "" { - f.Ctx.WriteString("Error: startTime cannot be empty") - UpdateLimitTriggerSa(nameStr, 2) - return - } - typeName := f.GetString("typeName") - if len(typeName) < 2 { - f.Ctx.WriteString("Error: typeName cannot be empty") - UpdateLimitTriggerSa(nameStr, 2) - return - } - legalBool := false - elt := models.EmailList{EmailType: 1} - el, eltErr := elt.Read("EmailType") - if eltErr != nil { - logs.Error("Failed to get mailing list, err: ", eltErr) - legalBool = true - } - for _, em := range el { - if strings.Contains(em.EmailName, "@") { - if strings.Split(em.EmailName, "@")[0] == typeName { - legalBool = true - } - } else { - legalBool = true - } - } - if !legalBool { - f.Ctx.WriteString("Error: The typeName parameter value is wrong") - UpdateLimitTriggerSa(nameStr, 2) - return - } - accessToken := os.Getenv("GITEE_TOKEN") - if accessToken == "" || len(accessToken) < 1 { - logs.Error("TriggerCveData, Issue token acquisition failed, "+ - "current time: ", common.GetCurTime()) - f.Ctx.WriteString("Error: Service internal error, try again later") - UpdateLimitTriggerSa(nameStr, 2) - return - } - // proc OpenEulerSaNum - taskhandler.UpdateUseOpenEulerSaNum() - rt := regexp.MustCompile(`^(\d{4})-\d{2}-(\d{2})$`) - find := rt.Match([]byte(startTime)) - if !find { - f.Ctx.WriteString(`Error: please enter the correct start time in a format like this "yyyy-MM-dd".`) - UpdateLimitTriggerSa(nameStr, 2) - return - } - cveNums := f.GetString("cveNums", "") - var cves = make([]string, 0) - if len(cveNums) != 0 { - cves = strings.Split(cveNums, ",") - } - //cve pull_request freeze time changed to a month ago - startTime = common.TimeStrSub(startTime, -180) - - dir := beego.AppConfig.DefaultString("fileDir", "download") - common.CreateDir(dir) - var wgCrvf sync.WaitGroup - componentMap := make(map[string]taskhandler.ComponentInfo) - cvrfFileList := make(map[string][]string) - cvrfFileMap := make(map[string]taskhandler.CvrfSa) - affectBranchSlice := make([]string, 0) - var unaffectcvrf taskhandler.UnaffectCvrfSa - cvrffileName := filepath.Join(dir, "cvrf-unaffected-cve-"+common.GetCurDate()+".xml") - cvrfFileSlice, cvrfOk := cvrfFileList[taskhandler.UNAFFECTCVRFKEY] - if cvrfOk && len(cvrfFileSlice) > 0 { - cvrffileName = cvrfFileSlice[0] - } else { - cvrfNameSlice := make([]string, 0) - cvrfNameSlice = append(cvrfNameSlice, cvrffileName) - cvrfFileList[taskhandler.UNAFFECTCVRFKEY] = cvrfNameSlice - } - updateInfoSlice := make([]taskhandler.UpdateInfoXml, 0) - go f.Ctx.WriteString("Processing: Data is being prepared, please wait patiently, Please check your mail later!\n") - unaffectcvrf.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" - unaffectcvrf.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" - du := beego.AppConfig.DefaultString("excel::v_pack_20_03_url", "") - csvPathList := strings.Split(du, ";") - if len(csvPathList) > 0 { - for _, csvP := range csvPathList { - openBranchx := strings.Split(csvP, "@") - if len(openBranchx) == 2 { - affectBranch := openBranchx[0] - affectBranchSlice = append(affectBranchSlice, affectBranch) - csvDownPath := openBranchx[1] - now := time.Now().Unix() - en := fmt.Sprintf("cve与安全公告%v_%v.xlsx", openBranchx[0], now) - fileCode := common.EncryptMd5(en) - wgCrvf.Add(1) - updateInfoxml := taskhandler.GenerateExcelTrigger(&wgCrvf, en, startTime, fileCode, affectBranch, - csvDownPath, dir, cvrfFileList, componentMap, cvrfFileMap, cves) - updateInfoSlice = append(updateInfoSlice, updateInfoxml) - } - } - } - wgCrvf.Wait() - taskhandler.WriteUnaffectCvrfXml(cvrffileName, &unaffectcvrf) - taskhandler.SaveCvrfName(cvrfFileList, cvrffileName, taskhandler.UNAFFECTCVRFKEY) - // Generate cvrf file - GenAndUploadCvrf(cvrfFileMap, cvrfFileList, componentMap, affectBranchSlice, dir, cves) - // Return the result first, continue processing the data - UpdateLimitTriggerSa(nameStr, 1) - GenUpdateInfoXmlFile(updateInfoSlice, dir, cves) -} - -// Generate updateinfo xml file -func GenUpdateInfoXmlFile(updateInfoSlice []taskhandler.UpdateInfoXml, dir string, cves []string) { - if len(updateInfoSlice) > 0 { - updateinfoDir := beego.AppConfig.String("obs::upload_updateinfo_dir") - var obsDir string - if len(cves) == 0 || cves == nil { - obsDir = updateinfoDir + common.GetCurDate() + "/" - } else { - obsDir = updateinfoDir + common.GetCurDate() + "-" + time.Now().Format("15-04-05") + "-specified" + "/" - } - dirErr, objectDir := taskhandler.ObsCreateDir(obsDir) - for _, upInfo := range updateInfoSlice { - // write excel - if len(upInfo.XmfilePath) > 1 { - fileNameList := make([]string, 0) - fileExt := filepath.Ext(upInfo.XmfilePath) - excelName := upInfo.XmfilePath[:(len(upInfo.XmfilePath)-len(fileExt))] + ".xlsx" - taskhandler.CreateUpdateExcel(excelName) - zipFileList := []string{upInfo.FileName, upInfo.XmfilePath} - fileNameList = append(fileNameList, excelName) - zipFileList = append(zipFileList, excelName) - // write xml - taskhandler.WriteXml(upInfo.XmfilePath, excelName, upInfo.AffectBranch, upInfo.Cvexml, - upInfo.Dpdates, upInfo.SecurityNotice, upInfo.PackRpmx) - // zip file - if isExist := taskhandler.FileExist(zipFileList); isExist { - zipFileName := upInfo.AffectBranch + "_" + common.GetCurDate() + "_" + strconv.Itoa(time.Now().Nanosecond()) + ".zip" - zipFileName = filepath.Join(dir, zipFileName) - zipErr := taskhandler.ZipFiles(zipFileName, zipFileList, dir, dir) - if zipErr != nil { - logs.Error("File compression failed: err: ", zipErr) - } - // send email - if len(cves) == 0 || cves == nil { - sendError := taskhandler.SendEmail(zipFileName, 0, "", "") - if sendError != nil { - logs.Error("SendEmail, sendErr: ", sendError) - return - } - } - zipFileList = append(zipFileList, zipFileName) - } - fileNameList = append(fileNameList, upInfo.XmfilePath) - // Upload to obs - if dirErr == nil && len(fileNameList) > 0 { - for _, fl := range fileNameList { - _, xmlFileName := filepath.Split(fl) - objectName := objectDir + xmlFileName - upErr := taskhandler.ObsUploadFile(objectName, fl) - if upErr != nil { - logs.Error("upErr: ", upErr) - } - } - } - // Clear file - taskhandler.DelFile(zipFileList) - } - } - } -} - -// Limit on the number of triggers -func LimitTriggerSa() (string, int) { - var captcha string - for i := 0; i < 6; i++ { - //产生0到9的整数 - num := rand.Intn(10) - //将整数转为字符串 - captcha += strconv.Itoa(num) - } - now := time.Now().UnixNano() - en := fmt.Sprintf("cve与安全公告%v_%v.xlsx", captcha, now) - timeUnix := time.Now().Unix() - //It is time-consuming to generate excel, here is the current limit processing - er := models.ExportRecord{} - err := er.QueryLast() - if err == nil && (timeUnix-er.CreateTime < 300) { - logs.Error("The number of requests is too frequent, please try again later, " + - "there is currently a task being processed") - return "", 1 - } else { - rand.Seed(now) - fileCode := common.EncryptMd5(en) - er = models.ExportRecord{FileName: en, FileCode: fileCode, State: 0, CreateTime: timeUnix} - err = er.Insert() - logs.Error("err: ", err) - } - return en, 0 -} - -func UpdateLimitTriggerSa(fileName string, state int8) { - fr := models.ExportRecord{FileName: fileName} - err := fr.Read("file_name") - if err != nil { - logs.Error("fr.Read, err: ", err) - return - } - fr.State = state - _ = fr.Update("state") -} - -// generate and upload cvrf.xml -func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, - cvrfFileList map[string][]string, componentMap map[string]taskhandler.ComponentInfo, - affectBranchSlice []string, dir string, cves []string) { - common.CreateDir(dir) - writeCvrfSlice := make([]string, 0) - totalFileSlice := make([]string, 0) - if len(componentMap) > 0 { - for comKey, comValue := range componentMap { - if len(affectBranchSlice) > 1 { - isEque := true - branchInfo1 := comValue.CveNumMap[affectBranchSlice[0]].CveNumSlice - for _, branchInfo := range affectBranchSlice[1:] { - if !common.CompareSlice(branchInfo1, comValue.CveNumMap[branchInfo].CveNumSlice) { - isEque = false - break - } - } - if isEque { - cvrfFileName := componentMap[comKey].CvrfFileName - cvrfVule := cvrfFileMap[cvrfFileName] - md5Ok, tmpOpenEulerSANum := taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, - componentMap[comKey].OpenEulerSANum, strings.Join(affectBranchSlice, ","), taskhandler.FIXEDFLAGE) - taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) - writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) - if md5Ok { - UpdateOpenEulerSaNumStatus(tmpOpenEulerSANum) - var saf models.SaFileList - saf.FileName = "cvrf-" + tmpOpenEulerSANum + ".xml" - models.DeleteCvrfFileName(&saf, "FileName") - } - for _, br := range comValue.CveNumMap { - UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) - } - } else { - for branch, br := range comValue.CveNumMap { - cvrfFileName := br.CvrfFileName - cvrfVule := cvrfFileMap[br.CvrfFileName] - md5Ok, tmpOpenEulerSANum := taskhandler.QueryCveMd5(br.CveNumSlice, componentMap[comKey].OwnedComponent, - br.OpenEulerSANum, branch, taskhandler.FIXEDFLAGE) - taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) - writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) - if md5Ok { - UpdateOpenEulerSaNumStatus(tmpOpenEulerSANum) - var saf models.SaFileList - saf.FileName = "cvrf-" + tmpOpenEulerSANum + ".xml" - models.DeleteCvrfFileName(&saf, "FileName") - } - } - UpdateOpenEulerSaNumStatus(comValue.OpenEulerSANum) - } - } else { - cvrfFileName := componentMap[comKey].CvrfFileName - cvrfVule := cvrfFileMap[cvrfFileName] - md5Ok, tmpOpenEulerSANum := taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, - componentMap[comKey].OpenEulerSANum, strings.Join(affectBranchSlice, ","), taskhandler.FIXEDFLAGE) - taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) - writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) - if md5Ok { - UpdateOpenEulerSaNumStatus(tmpOpenEulerSANum) - var saf models.SaFileList - saf.FileName = "cvrf-" + tmpOpenEulerSANum + ".xml" - models.DeleteCvrfFileName(&saf, "FileName") - } - for _, br := range comValue.CveNumMap { - UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) - } - } - } - } - cvrfFileList[taskhandler.CVRFFKEY] = writeCvrfSlice - uploadCvrfFile(cvrfFileList, totalFileSlice, dir, componentMap, cves) -} - -func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, dir string, - componentMap map[string]taskhandler.ComponentInfo, cves []string) { - uploadCvrfDir := beego.AppConfig.String("obs::upload_cvrf_dir") - downloadCvrfDir := beego.AppConfig.String("obs::download_cvrf_dir") - var obsDir string - if len(cves) == 0 || cves == nil { - obsDir = uploadCvrfDir + common.GetCurDate() + "/" - } else { - obsDir = uploadCvrfDir + common.GetCurDate() + "-" + time.Now().Format("15-04-05") + "-specified" + "/" - } - dirErr, objectDir := taskhandler.ObsCreateDir(obsDir) - if dirErr != nil { - logs.Error("dirErr: ", dirErr) - } - - // set upload dirname of hotpatch - obsimpl.SetDynamicDir(obsDir) - - indexFilePath := filepath.Join(dir, "index.txt") - indexObjectName := downloadCvrfDir + "index.txt" - downObsErr := taskhandler.ObsDownloadFile(indexObjectName, indexFilePath) - if downObsErr != nil { - logs.Error("downObsErr: ", downObsErr) - return - } - saFileStr := taskhandler.ReadFileAll(indexFilePath) - SaveFileToDb(saFileStr) - totalSlice := make([]string, 0) - saNumber := int64(1000) - sfl := models.GetCvrfAllFile(models.SaFileRecordSa) - curYears := strconv.Itoa(time.Now().Year()) - var oldyear string - saDir := beego.AppConfig.DefaultString("saFileDir", "download/sa") - common.CreateAllDir(saDir) - if len(sfl) > 0 { - for _, l := range sfl { - fSlice := strings.Split(l.FileName, "-") - dirYears := curYears - if len(fSlice) > 4 { - dirYears = fSlice[3] - oldyear = fSlice[3] - } - totalSlice = append(totalSlice, dirYears+"/"+l.FileName) - saNumber = l.SaNumber - } - } - if len(oldyear) > 0 && curYears > oldyear { - saNumber = int64(1000) - } - // File name and data stored in database - fileSlice, fOk := cvrfFileList[taskhandler.CVRFFKEY] - if fOk { - totalFileSlice = UploadSaFile(fileSlice, totalSlice, totalFileSlice, saNumber, - curYears, saDir, dir, objectDir) - } - unaffFileSlice, unffOk := cvrfFileList[taskhandler.UNAFFECTCVRFKEY] - if unffOk { - subFileSlice := make([]string, 0) - updateFilePath := filepath.Join(dir, "update_unaffect.txt") - for _, fPath := range unaffFileSlice { - //taskhandler.WriteUnaffectCvrfXml(fPath, componentMap[fPath].UnaffectFile) - _, fileName := filepath.Split(fPath) - // File storage to db - recordErr := taskhandler.RecordCrvfInfo(fPath, fileName, "", taskhandler.UNAFFECTFLAG) - logs.Info("recordErr: ", recordErr) - // Upload file, pending - subFileSlice = append(subFileSlice, fileName) - totalFileSlice = append(totalFileSlice, fPath) - // Upload successfully, modify file status - if len(fileName) > 5 { - openEulerSANum := fileName[5 : len(fileName)-4] - taskhandler.UpdateCvrfRecord(openEulerSANum, 2) - } - } - taskhandler.ReadWriteFile(updateFilePath, subFileSlice) - totalFileSlice = append(totalFileSlice, updateFilePath) - } - if len(totalFileSlice) > 0 { - for _, localFilePath := range totalFileSlice { - _, localFileName := filepath.Split(localFilePath) - obsFilePath := objectDir + localFileName - obsErr := taskhandler.PostFile(localFilePath, obsFilePath) - if obsErr != nil { - logs.Error("obsErr: ", obsErr) - } - } - dir := "download" - zipFileName := "cvrf-" + common.GetCurDate() + "_" + strconv.Itoa(time.Now().Nanosecond()) + ".zip" - zipFileName = filepath.Join(dir, zipFileName) - zipErr := taskhandler.ZipFiles(zipFileName, totalFileSlice, dir, dir) - if zipErr != nil { - logs.Error("File compression failed: err: ", zipErr) - } - // send email - if len(cves) == 0 || cves == nil { - sendError := taskhandler.SendEmail(zipFileName, 1, "", "") - if sendError != nil { - logs.Error("SendEmail, sendErr: ", sendError) - return - } - } - totalFileSlice = append(totalFileSlice, zipFileName) - } - // Delete local files - taskhandler.DelFile(totalFileSlice) - logs.Info(cvrfFileList[taskhandler.BRANCHSKEY], ",End of generating cvrf format file this time") -} - -func UploadSaFile(fileSlice, totalSlice, totalFileSlice []string, saNumber int64, - curYears, saDir, dir, uploadPath string) []string { - subFileSlice := make([]string, 0) - oldFileSlice := make([]string, 0) - for _, fPath := range fileSlice { - _, fileName := filepath.Split(fPath) - oldSa := RegSa(fileName) - saNumber = saNumber + 1 - oldText := fmt.Sprintf("openEuler-SA-%v-%v", curYears, oldSa) - newText := fmt.Sprintf("openEuler-SA-%v-%v", curYears, saNumber) - newFile := fmt.Sprintf("cvrf-%v.xml", newText) - newPath := filepath.Join(saDir, newFile) - ReplaceFileSa(fPath, newPath, oldText, newText) - oldFileSlice = append(oldFileSlice, fPath) - // File storage to db - recordErr := taskhandler.RecordCrvfInfo(fPath, fileName, newText, taskhandler.FIXEDFLAGE) - logs.Info("recordErr: ", recordErr) - // Upload file, pending - SaveFileRecord(newFile) - totalSlice = append(totalSlice, curYears+"/"+newFile) - taskhandler.ProcCvrfFileName(fileName) - subFileSlice = append(subFileSlice, curYears+"/"+newFile) - totalFileSlice = append(totalFileSlice, newPath) - // Upload successfully, modify file status - if len(newFile) > 5 { - openEulerSANum := newFile[5 : len(newFile)-4] - taskhandler.UpdateCvrfRecord(openEulerSANum, 2) - } - } - indexFilePath := filepath.Join(dir, "index.txt") - updateFilePath := filepath.Join(dir, "update_fixed.txt") - readErr := taskhandler.ReadWriteFile(indexFilePath, totalSlice) - if readErr != nil { - logs.Error(indexFilePath, readErr) - } - readErr = taskhandler.ReadWriteFile(updateFilePath, subFileSlice) - if readErr != nil { - logs.Error(updateFilePath, readErr) - } - totalFileSlice = append(totalFileSlice, indexFilePath) - totalFileSlice = append(totalFileSlice, updateFilePath) - taskhandler.DelFile(oldFileSlice) - return totalFileSlice -} - -func UpdateOpenEulerSaNumStatus(openEulerSANum string) { - var sa models.SaNumber - curYears := strconv.Itoa(time.Now().Year()) - sa.SaYears = curYears - sa.OpenEulerSANum = openEulerSANum - sa.UpdateTime = common.GetCurTime() - getErr := models.GetSaNumber(&sa, "OpenEulerSANum", "SaYears") - if getErr == nil { - sa.Status = 1 - models.UpdateSaNumber(&sa, "status") - } -} - -func RegSa(bufVule string) string { - subSlice1 := strings.Split(bufVule, ".") - if len(subSlice1) > 0 { - subSlice2 := strings.Split(subSlice1[0], "-") - if len(subSlice2) > 0 { - result := subSlice2[len(subSlice2)-1] - return result - } - } - return "" -} - -func SaveFileToDb(saFileStr string) { - if len(saFileStr) > 1 { - saFileSlice := strings.Split(saFileStr, "\n") - if len(saFileSlice) > 0 { - models.DeleteCvrfFileRecord() - for _, saf := range saFileSlice { - if len(saf) > 2 { - singSlice := strings.Split(saf, "/") - if len(singSlice) > 1 { - SaveFileRecord(singSlice[1]) - } - } - } - } - } -} - -func SaveFileRecord(fileName string) { - var af models.SaFileRecord - result2 := RegSa(fileName) - saNumber, _ := strconv.ParseInt(result2, 10, 64) - af.SaNumber = saNumber - af.Status = 1 - af.FileName = fileName - af.CreateTime = common.GetCurTime() - af.SaType = af.GenSaType() - - models.InsertCvrfFileRecord(&af) -} - -func ReplaceFileSa(oldPath, newPath, oldText, newText string) { - helper := taskhandler.ReplaceHelper{ - OldPath: oldPath, - NewPath: newPath, - OldText: oldText, - NewText: newText, - } - err := helper.DoWrok() - if err == nil { - fmt.Println("done!") - } else { - fmt.Println("error:", err.Error()) - } -} - -func (f *FileController) TriggerUnaffectedCve() { - startTime := f.GetString("startTime") - if startTime == "" { - f.Ctx.WriteString("Error: startTime cannot be empty") - return - } - accessToken := os.Getenv("GITEE_TOKEN") - - if accessToken == "" || len(accessToken) < 1 { - logs.Error("TriggerUnaffectedCve, Issue token acquisition failed, current time: ", common.GetCurTime()) - f.Ctx.WriteString("Error: Service internal error, try again later") - return - } - owner := beego.AppConfig.String("gitee::owner") - - unaffectYear, yerr := beego.AppConfig.Int("excel::unaffect_year") - if yerr != nil { - unaffectYear = 2018 - } - dir := beego.AppConfig.DefaultString("fileDir", "download") - _ = common.CreateDir(dir) - var unaffectcvrf taskhandler.UnaffectCvrfSa - cvrffileName := filepath.Join(dir, "cvrf-unaffected-cve-"+common.GetCurDate()+".xml") - unaffectcvrf.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" - unaffectcvrf.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" - du := beego.AppConfig.DefaultString("excel::v_pack_20_03_url", "") - csvPathList := strings.Split(du, ";") - if len(csvPathList) > 0 { - for _, csvP := range csvPathList { - openBranchx := strings.Split(csvP, "@") - if len(openBranchx) == 2 { - affectBranch := openBranchx[0] - taskhandler.UnaffectIssueProc(affectBranch, nil, nil, startTime, accessToken, owner, &unaffectcvrf, unaffectYear, nil) - } - } - } - writeXml(cvrffileName, unaffectcvrf) - uploadUnaffected(cvrffileName) - taskhandler.DelFile([]string{cvrffileName}) -} - -func writeXml(path string, unaffectcvrf taskhandler.UnaffectCvrfSa) { - _ = os.Remove(path) - xmlOutPut, outPutErr := xml.MarshalIndent(unaffectcvrf, "", " ") - if outPutErr == nil { - headerBytes := []byte(xml.Header) - xmlOutPutData := append(headerBytes, xmlOutPut...) - _ = ioutil.WriteFile(path, xmlOutPutData, os.ModePerm) - } else { - logs.Error(outPutErr) - } -} - -func uploadUnaffected(path string) { - uploadCvrfDir := beego.AppConfig.String("obs::upload_cvrf_dir") - obsDir := uploadCvrfDir + common.GetCurDate() + "-unaffected/" - dirErr, objectDir := taskhandler.ObsCreateDir(obsDir) - if dirErr != nil { - logs.Error("dirErr: ", dirErr) - } - - _, localFileName := filepath.Split(path) - obsFilePath := objectDir + localFileName - obsErr := taskhandler.PostFile(path, obsFilePath) - if obsErr != nil { - logs.Error("obsErr: ", obsErr) - } -} diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go index a79d7af5759e72efe42c543b0867de28a197b6c4..ab5df36df40309d18225dd17521ab53c547e603e 100644 --- a/cve-vulner-manager/controllers/hook.go +++ b/cve-vulner-manager/controllers/hook.go @@ -195,6 +195,10 @@ func (c *HookEventControllers) handleIssue() { nameSpace := util.TrimString(issueHook.Repository.NameSpace) organizationID := int8(1) organizationID = taskhandler.GetOrganizationId(nameSpace) + if organizationID != 1 { + return + } + hookPwd := beego.AppConfig.String("hook::hookpwd") issueHook.Password = util.TrimString(issueHook.Password) if issueHook.Password != hookPwd { @@ -1453,11 +1457,14 @@ func handleIssueComment(payload models.CommentPayload) { path = beego.AppConfig.String("opengauss::gauss_issue_path") } cBody = strings.ReplaceAll(cBody, util.KwOpenGaussScore, util.KwOpenEulerScore) + return } else if vc.OrganizationID == 3 { // Query the repo that needs to submit an issue cBody = strings.ReplaceAll(cBody, util.KwMindSporeScore, util.KwOpenEulerScore) + return } else if vc.OrganizationID == 4 { cBody = strings.ReplaceAll(cBody, util.KwLooKengScore, util.KwOpenEulerScore) + return } if len(repoPath) > 1 && repoPath != path { vc.PackName = repoPath @@ -2431,8 +2438,6 @@ func SetIssueStateByReason(issue *models.IssueTemplate, remoteStateName string) issue.Status = 3 } - models.UpdateIssueTemplate(issue, "Status", "StatusName") - // issue本身的状态和计算后的状态一致，则不做任何操作 stateNameMap := common.GetStateNameMap() stateName, ok := stateNameMap[state] diff --git a/cve-vulner-manager/controllers/upload.go b/cve-vulner-manager/controllers/upload.go index 6c806b11ea61add1fbaf36ab4cb8103e15aceafa..d7d5660fc45bf27bcc53928f5f30c1f9ded56438 100644 --- a/cve-vulner-manager/controllers/upload.go +++ b/cve-vulner-manager/controllers/upload.go @@ -303,6 +303,11 @@ func AddOrgUpstream(source int, CveDataDict common.CveOriginData) (ResDataList [ orCve.Version = version orCve.UpdateType = updateType orCve.CvePackName = cvePackName + packageTime, err := json.Marshal(CveDataDict.PackageTime) + if err != nil { + logs.Error("marshal packageTime failed,", err.Error()) + } + orCve.PackageTime = string(packageTime) orCve.Credibility = CveDataDict.Credibility //if CveDataDict.Credibility == 7 { // rejectIssue(cveNum) @@ -422,7 +427,7 @@ func AddOrgUpstream(source int, CveDataDict common.CveOriginData) (ResDataList [ orCve.AffectUpdate = 1 } - _, err := models.CreateOriginCve(CveDataDict, &orCve, &od, &ous, &osi, &osp, ose, osv, &osf, packageUrls) + _, err = models.CreateOriginCve(CveDataDict, &orCve, &od, &ous, &osi, &osp, ose, osv, &osf, packageUrls) if err == nil { logs.Info("Cve original data is successfully created CveNum: ", CveDataDict.Ids) ResData.CveNum = CveDataDict.Ids diff --git a/cve-vulner-manager/models/cve.go b/cve-vulner-manager/models/cve.go index 2692a66dd15234d65c424bbf19d0d3da70d7274f..7ec934a16f3e281aa7a52dfbba51e62c75abf424 100644 --- a/cve-vulner-manager/models/cve.go +++ b/cve-vulner-manager/models/cve.go @@ -1,6 +1,7 @@ package models import ( + "encoding/json" "errors" "fmt" "strconv" @@ -175,10 +176,12 @@ func QueryCveDesc(cveId int64) (OriginUpstreamDesc, bool) { } } -func QueryCveImpact(cveId int64) (OriginUpstreamImpact, bool) { +// QueryCveImpact get impact by cveid and source +func QueryCveImpact(cveId int64, source string) (OriginUpstreamImpact, bool) { o := orm.NewOrm() var cveImpact OriginUpstreamImpact - err := o.Raw("select * from cve_origin_upstream_impact where cve_id = ?", cveId).QueryRow(&cveImpact) + err := o.Raw("select * from cve_origin_upstream_impact where cve_id = ? and source = ?", cveId, source). + QueryRow(&cveImpact) if err != nil { logs.Error(cveId, "QueryCveImpact, cve_origin_upstream_impact, cveId: ", cveId, ",err: ", err) return cveImpact, false @@ -1052,7 +1055,7 @@ func GetCanExportExcelData(cveNum, issueNum, repo string, issueId int64) (list [ if cveNum == "" { return list, errors.New("cve number can not empty") } - sql := `SELECT b.num,c.*,a.issue_num,a.owned_component,a.cve_brief, + sql := `SELECT b.num,a.issue_num,a.owned_component,a.cve_brief,a.cve_num,a.openeuler_score,a.openeuler_vector, d.sec_id,d.introduction,d.summary,d.theme,d.description,d.influence_component, d.affect_product,d.reference_link,d.affect_status, e.public_date,e.openeuler_sa_num,a.cve_level,b.organizate_id,a.affected_version,a.analysis_version,a.issue_label @@ -1062,8 +1065,6 @@ RIGHT JOIN bc.cve_id,bc.cve_num,bc.organizate_id FROM cve_vuln_center bc WHERE bc.cve_num = ? AND bc.is_export in (0,3) AND bc.pack_name = ? AND bc.organizate_id = 1) b ON a.cve_id = b.cve_id -LEFT JOIN cve_score c -ON b.cve_id = c.cve_id LEFT JOIN cve_security_notice d ON b.cve_id = d.cve_id LEFT JOIN cve_open_euler_s_a e @@ -1075,27 +1076,6 @@ WHERE a.issue_num = ? and a.issue_id = ? and b.organizate_id = 1 return } -func GetCanExportCveDataSameNum(cId string) (list []ExcelExport, err error) { - sql := `SELECT a.cve_id,a.owned_component,a.cve_brief,c.*,d.sec_id, -d.introduction,d.summary,d.theme,d.description,d.influence_component, -d.affect_product,d.reference_link,d.affect_status, -e.public_date,e.openeuler_sa_num,a.cve_level -FROM cve_issue_template a -LEFT JOIN cve_score c -ON c.cve_id = a.cve_id -LEFT JOIN cve_security_notice d -ON d.cve_id = a.cve_id -LEFT JOIN cve_open_euler_s_a e -ON e.cve_id = a.cve_id -WHERE a.cve_id IN (%s) -ORDER BY c.openeuler_score DESC -` - sql = fmt.Sprintf(sql, cId) - o := orm.NewOrm() - _, err = o.Raw(sql).QueryRows(&list) - return -} - func GetIssueNumber(packName string, cves []string) (issueTemp []IssueTemplate, err error) { var sql string if len(cves) == 0 { @@ -1260,6 +1240,7 @@ type CveAllIssueData struct { IssueCreate string `orm:"column(issue_create)"` IsIgnore int8 `orm:"column(is_ignore)"` AffectUpdate int8 `orm:"column(affect_update)"` + PackageTime string `orm:"column(package_time)"` } // QueryIssue query issuewhitelist @@ -1268,7 +1249,8 @@ func QueryCveAllIssueData(currentPage, pageSize, communityFlag int, startTime st o := orm.NewOrm() var sql = `SELECT t.cve_num, t.nvd_score, t.openeuler_score, t.issue_num,t.status, t.affected_version, t.owned_version,t.is_ignore,t.owned_component,t.owner,t.repo,t.cve_level,t.create_time,v.repair_time, - s.sa_release_time,s.rpm_release_time,v.first_per_time,v.first_get_time,v.affect_update,o.issue_create + s.sa_release_time,s.rpm_release_time,v.first_per_time,v.first_get_time,v.affect_update,v.package_time, + o.issue_create FROM cve_vuln_center v,cve_issue_template t left join cve_issue_template_association s on t.template_id = s.template_id left join cve_gite_origin_issue o on t.issue_num = o.number and t.issue_id = o.issue_id where t.status != 6 and t.cve_id = v.cve_id %s order by t.update_time desc limit ? offset ?` @@ -1479,3 +1461,19 @@ func QueryPackageByBranch(branch string) ([]OpenEulerRepoOrigin, error) { return data, err } + +// GetPackageTime get package time by name +func (u *OriginUpstream) GetPackageTime(packageName string) string { + var packageTime []common.PackageTime + if err := json.Unmarshal([]byte(u.PackageTime), &packageTime); err != nil { + return "" + } + + for _, v := range packageTime { + if v.Name == packageName { + return v.AddTime + } + } + + return "" +} diff --git a/cve-vulner-manager/models/cvedetail.go b/cve-vulner-manager/models/cvedetail.go index 1e5cf854ad6c982c12ebd9fd45de4aa4c2b5e35b..e19ac36dd633bd5c20be555908a697a5dd84825d 100644 --- a/cve-vulner-manager/models/cvedetail.go +++ b/cve-vulner-manager/models/cvedetail.go @@ -20,7 +20,7 @@ LEFT JOIN cve_origin_upstream_config c ON a.cve_id = c.cve_id LEFT JOIN cve_origin_upstream_impact d ON a.cve_id = d.cve_id LEFT JOIN cve_origin_upstream_poc e ON a.cve_id = e.cve_id LEFT JOIN cve_origin_upstream_fix_suggest f ON a.cve_id = f.cve_id -WHERE a.cve_num = ? ORDER BY a.cve_id DESC`, cveNum, +WHERE a.cve_num = ? and d.source = "nvd" ORDER BY a.cve_id DESC`, cveNum, ).QueryRows(&vs) if err != nil { logs.Error("GetOriginUpstream, cveNum: ", cveNum, ", Query failed") @@ -58,7 +58,7 @@ LEFT JOIN cve_origin_upstream_config c ON a.cve_id = c.cve_id LEFT JOIN cve_origin_upstream_impact d ON a.cve_id = d.cve_id LEFT JOIN cve_origin_upstream_poc e ON a.cve_id = e.cve_id LEFT JOIN cve_origin_upstream_fix_suggest f ON a.cve_id = f.cve_id -WHERE a.source = 0 ORDER BY a.cve_id DESC LIMIT ? OFFSET ?`, +WHERE a.source = 0 and d.source = "nvd" ORDER BY a.cve_id DESC LIMIT ? OFFSET ?`, pageSize, startSize, ).QueryRows(&v) if err != nil { @@ -244,46 +244,6 @@ func (ou OriginUpstreamJoinData) ToDetailSummary() (cod common.CveOriginData) { } cod.Configurations = cf - cveScore, _ := QueryCveScore(ou.ImpactId, "v3") - if cveScore.ScoreId > 0 { - cveScV3, _ := QueryCveCvssV3(cveScore.ScoreId) - cod.Impact.BaseMetricV3.CvssV3.Version = cveScV3.Version - cod.Impact.BaseMetricV3.CvssV3.VectorString = cveScV3.VectorString - cod.Impact.BaseMetricV3.CvssV3.AttackComplexity = cveScV3.AttackComplexity - cod.Impact.BaseMetricV3.CvssV3.AttackVector = cveScV3.AttackVector - cod.Impact.BaseMetricV3.CvssV3.AvailabilityImpact = cveScV3.AvailabilityImpact - cod.Impact.BaseMetricV3.CvssV3.BaseSeverity = cveScV3.BaseSeverity - cod.Impact.BaseMetricV3.CvssV3.UserInteraction = cveScV3.UserInteraction - cod.Impact.BaseMetricV3.CvssV3.BaseScore = cveScV3.BaseScore - cod.Impact.BaseMetricV3.CvssV3.PrivilegesRequired = cveScV3.PrivilegesRequired - cod.Impact.BaseMetricV3.CvssV3.ConfidentialityImpact = cveScV3.ConfidentialityImpact - cod.Impact.BaseMetricV3.CvssV3.IntegrityImpact = cveScV3.IntegrityImpact - cod.Impact.BaseMetricV3.CvssV3.Scope = cveScV3.Scope - cod.Impact.BaseMetricV3.ExploitabilityScore = cveScV3.ExploitabilityScore - cod.Impact.BaseMetricV3.ImpactScore = cveScV3.ImpactScore - } - cveScoreV2, _ := QueryCveScore(ou.ImpactId, "v2") - if cveScoreV2.ScoreId > 0 { - cveScV2, _ := QueryCveCvssV2(cveScoreV2.ScoreId) - cod.Impact.BaseMetricV2.CvssV2.VectorString = cveScV2.VectorString - cod.Impact.BaseMetricV2.CvssV2.AccessComplexity = cveScV2.AccessComplexity - cod.Impact.BaseMetricV2.CvssV2.AvailabilityImpact = cveScV2.AvailabilityImpact - cod.Impact.BaseMetricV2.CvssV2.Authentication = cveScV2.Authentication - cod.Impact.BaseMetricV2.CvssV2.Version = cveScV2.Version - cod.Impact.BaseMetricV2.CvssV2.BaseScore = cveScV2.BaseScore - cod.Impact.BaseMetricV2.CvssV2.IntegrityImpact = cveScV2.IntegrityImpact - cod.Impact.BaseMetricV2.CvssV2.ConfidentialityImpact = cveScV2.ConfidentialityImpact - cod.Impact.BaseMetricV2.CvssV2.AccessVector = cveScV2.AccessVector - cod.Impact.BaseMetricV2.AcInsufInfo = cveScV2.AcInsufInfo - cod.Impact.BaseMetricV2.UserInteractionRequired = cveScV2.UserInteractionRequired - cod.Impact.BaseMetricV2.Severity = cveScV2.Severity - cod.Impact.BaseMetricV2.ObtainUserPrivilege = cveScV2.ObtainUserPrivilege - cod.Impact.BaseMetricV2.ObtainAllPrivilege = cveScV2.ObtainAllPrivilege - cod.Impact.BaseMetricV2.ImpactScore = cveScV2.ImpactScore - cod.Impact.BaseMetricV2.ExploitabilityScore = cveScV2.ExploitabilityScore - cod.Impact.BaseMetricV2.ObtainOtherPrivilege = cveScV2.ObtainOtherPrivilege - } - cod.Poc.Url = ou.Url cod.Poc.Date = ou.Date cod.Poc.Source = ou.Spoc diff --git a/cve-vulner-manager/models/excel.go b/cve-vulner-manager/models/excel.go index 34e8f9c6459027c12299cec2da17b5ef145ccc02..a19889de8532ea6f3c876e6b6edd641bfc029501 100644 --- a/cve-vulner-manager/models/excel.go +++ b/cve-vulner-manager/models/excel.go @@ -14,27 +14,29 @@ const splitLen = 2 // ExcelExport the export excel row content model type ExcelExport struct { - Num int64 - Score - CveBrief string `json:"cve_brief"` - OwnedComponent string `json:"owned_component"` - SecID int64 `json:"sec_id" orm:"column(sec_id)"` - Introduction string `json:"introduction"` - Summary string `json:"summary"` - Theme string `json:"theme"` - Description string `json:"description"` - InfluenceComponent string `json:"influence_component" orm:"size(256);null;column(influence_component)"` - AffectProduct string `json:"affect_product"` - ReferenceLink string `json:"reference_link"` - AffectStatus string `json:"affect_status"` - PublicDate string `json:"public_date"` - OpenEulerSANum string `json:"openeuler_sa_num" orm:"size(128);column(openeuler_sa_num)"` - CveLevel string `json:"cve_level" orm:"size(32);column(cve_level)"` - OrganizateId int8 `json:"organizate_id" orm:"column(organizate_id)"` - AffectedVersion string `json:"affected_version" orm:"column(affected_version)"` - AnalysisVersion string `json:"analysis_version" orm:"column(analysis_version)"` - IssueLabel string `json:"issue_label" orm:"column(issue_label)"` - Repo string `json:"repo" orm:"column(repo)"` + Num int64 + CveNum string `json:"cve_num"` + OpeneulerScore float64 `json:"openeuler_score"` + OpeneulerVector string `json:"openeuler_vector"` + CveBrief string `json:"cve_brief"` + OwnedComponent string `json:"owned_component"` + SecID int64 `json:"sec_id" orm:"column(sec_id)"` + Introduction string `json:"introduction"` + Summary string `json:"summary"` + Theme string `json:"theme"` + Description string `json:"description"` + InfluenceComponent string `json:"influence_component" orm:"size(256);null;column(influence_component)"` + AffectProduct string `json:"affect_product"` + ReferenceLink string `json:"reference_link"` + AffectStatus string `json:"affect_status"` + PublicDate string `json:"public_date"` + OpenEulerSANum string `json:"openeuler_sa_num" orm:"size(128);column(openeuler_sa_num)"` + CveLevel string `json:"cve_level" orm:"size(32);column(cve_level)"` + OrganizateId int8 `json:"organizate_id" orm:"column(organizate_id)"` + AffectedVersion string `json:"affected_version" orm:"column(affected_version)"` + AnalysisVersion string `json:"analysis_version" orm:"column(analysis_version)"` + IssueLabel string `json:"issue_label" orm:"column(issue_label)"` + Repo string `json:"repo" orm:"column(repo)"` } func (e ExcelExport) IsIssueWithAnalysisVersion() bool { diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index d9be60f6a556e686de7c3d3d72748854975f560a..627e58ac3b307e5f70437cc5b2294fe631d6abf6 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -14,6 +14,8 @@ const ( OrganizationIdOpengauss = 2 OrganizationIdMindspore = 3 OrganizationIdOpenLooKeng = 4 + + IssueStatusRejected = 4 ) type AuthTokenInfo struct { @@ -75,6 +77,7 @@ type VulnCenter struct { CveDetailUrl string `orm:"type(text);column(cve_detail_url)" description:"cve详情链接"` OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:来源mindspore;4:来源openLooKeng"` AffectUpdate int8 `orm:"default(0);column(affect_update)" description:"表示影响组件内容是否更新，1为有更新，0为无更新"` + PackageTime string `orm:"size(512);column(package_time)" description:"包的首次记录时间，majun数据统计用"` CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` @@ -334,6 +337,7 @@ type OriginUpstream struct { UpdateType string `orm:"size(32);column(update_type);" description:"数据上传类型：insert, update, delete"` CvePackName string `orm:"size(512);column(cve_packname);null" description:"Cve在上游对应的包名"` PackName string `orm:"size(512);column(git_packname);index;null" description:"Cve对应的openEuler包名称(或者影响的包名)"` + PackageTime string `orm:"size(512);column(package_time)" description:"包的首次记录时间，majun统计数据用"` Title string `orm:"type(text);column(cve_title);null" description:"标题"` AffectProduct string `orm:"size(512);column(affect_porduct);null" description:"Cve影响的组件, 对应"` CnnvdID string `orm:"size(256);column(cnnvd_id);null" description:"Cnnvd_id"` @@ -414,6 +418,7 @@ type OriginUpstreamImpact struct { ImpactId int64 `orm:"pk;auto;column(impact_id)"` CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` Impact string `orm:"size(32);column(impact);null" description:"包含v2,v3评分数据"` + Source string `orm:"size(256);column(source);default(nvd)" description:"来源"` } type OriginUpstreamPatch struct { diff --git a/cve-vulner-manager/models/uploadcve.go b/cve-vulner-manager/models/uploadcve.go index 2f7ad5a8bb21c4c44aa0cfd614876f9d60e8fa50..8bd070b98f8b0433fccbb11bad0f45ed44a4d1e5 100644 --- a/cve-vulner-manager/models/uploadcve.go +++ b/cve-vulner-manager/models/uploadcve.go @@ -314,131 +314,8 @@ func CreateOriginCve(CveData common.CveOriginData, ou *OriginUpstream, od *Origi } } } - osi.CveId = num - losi := OriginUpstreamImpact{CveId: num} - errxx := o.Read(&losi, "CveId") - if errxx == orm.ErrNoRows || errxx == orm.ErrMissPK { - logs.Info("CreateOriginCve, cve_origin_upstream_impact, "+ - "Data does not exist, cveId: ", num) - } else { - var losis []OriginUpstreamImpactScore - losisNum, err := o.Raw("select * from cve_origin_upstream_impact_score where impact_id = ?", - losi.ImpactId).QueryRows(&losis) - if err != nil { - logs.Info("CreateOriginCve, cve_origin_upstream_impact_score, Data does not exist,"+ - " losi.ImpactId: ", losi.ImpactId, ", err: ", err, ",losisNum: ", losisNum) - } else { - for _, sis := range losis { - if sis.CvssV3 == 1 && sis.BaseMetricV3 == 1 { - lousisv3 := OriginUpstreamImpactScoreV3{ScoreId: sis.ScoreId} - o.Delete(&lousisv3, "ScoreId") - } - if sis.CvssV2 == 1 && sis.BaseMetricV2 == 1 { - lousisv2 := OriginUpstreamImpactScoreV2{ScoreId: sis.ScoreId} - o.Delete(&lousisv2, "ScoreId") - } - } - losisx := OriginUpstreamImpactScore{ImpactId: losi.ImpactId} - o.Delete(&losisx, "ImpactId") - } - losix := OriginUpstreamImpact{CveId: num} - o.Delete(&losix, "CveId") - } - if losinum, err := o.Insert(osi); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact success, lousnum:", - losinum, ", cveNum", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact failed,"+ - " Lnode:", osi, ", err: ", err) - o.Rollback() - return 0, err - } - var lousist OriginUpstreamImpactScore - lousist.ImpactId = osi.ImpactId - lousist.BaseMetricV3 = 1 - lousist.BaseMetricV2 = 0 - lousist.CvssV3 = 1 - lousist.CvssV2 = 0 - lousist.Status = 1 - if lousistnum, err := o.Insert(&lousist); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score success, "+ - "lousistnum:", lousistnum, ", cveNum", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score failed, "+ - "lousist:", lousist, ", err: ", err) - o.Rollback() - return 0, err - } - var lousisv3 OriginUpstreamImpactScoreV3 - lousisv3.ScoreId = lousist.ScoreId - lousisv3.BaseScore = CveData.Impact.BaseMetricV3.CvssV3.BaseScore - lousisv3.VectorString = CveData.Impact.BaseMetricV3.CvssV3.VectorString - lousisv3.AttackComplexity = CveData.Impact.BaseMetricV3.CvssV3.AttackComplexity - lousisv3.AttackVector = CveData.Impact.BaseMetricV3.CvssV3.AttackVector - lousisv3.AvailabilityImpact = CveData.Impact.BaseMetricV3.CvssV3.AvailabilityImpact - lousisv3.BaseSeverity = CveData.Impact.BaseMetricV3.CvssV3.BaseSeverity - lousisv3.UserInteraction = CveData.Impact.BaseMetricV3.CvssV3.UserInteraction - lousisv3.PrivilegesRequired = CveData.Impact.BaseMetricV3.CvssV3.PrivilegesRequired - lousisv3.Version = CveData.Impact.BaseMetricV3.CvssV3.Version - lousisv3.ConfidentialityImpact = CveData.Impact.BaseMetricV3.CvssV3.ConfidentialityImpact - lousisv3.IntegrityImpact = CveData.Impact.BaseMetricV3.CvssV3.IntegrityImpact - lousisv3.Scope = CveData.Impact.BaseMetricV3.CvssV3.Scope - lousisv3.ImpactScore = CveData.Impact.BaseMetricV3.ImpactScore - lousisv3.ExploitabilityScore = CveData.Impact.BaseMetricV3.ExploitabilityScore - lousisv3.CveLevel = OpenEulerScoreProc(CveData.Impact.BaseMetricV3.CvssV3.BaseScore) - if lousistnumv3, err := o.Insert(&lousisv3); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score_v3 success, "+ - "lousistnumv3:", lousistnumv3, ", cveNum: ", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score_v3 failed, "+ - "lousisv3:", lousisv3, ", err: ", err) - o.Rollback() - return 0, err - } - var lousistv2 OriginUpstreamImpactScore - lousistv2.ImpactId = osi.ImpactId - lousistv2.BaseMetricV3 = 0 - lousistv2.BaseMetricV2 = 1 - lousistv2.CvssV3 = 0 - lousistv2.CvssV2 = 1 - lousistv2.Status = 1 - if lousistnum, err := o.Insert(&lousistv2); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score success, "+ - "lousistnum:", lousistnum, ", cveNum: ", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score failed, "+ - "lousistv2:", lousistv2, ", err: ", err) - o.Rollback() - return 0, err - } - var lousisv2 OriginUpstreamImpactScoreV2 - lousisv2.ScoreId = lousistv2.ScoreId - lousisv2.BaseScore = CveData.Impact.BaseMetricV2.CvssV2.BaseScore - lousisv2.AcInsufInfo = CveData.Impact.BaseMetricV2.AcInsufInfo - lousisv2.BaseScore = CveData.Impact.BaseMetricV2.CvssV2.BaseScore - lousisv2.VectorString = CveData.Impact.BaseMetricV2.CvssV2.VectorString - lousisv2.AccessComplexity = CveData.Impact.BaseMetricV2.CvssV2.AccessComplexity - lousisv2.Authentication = CveData.Impact.BaseMetricV2.CvssV2.Authentication - lousisv2.AvailabilityImpact = CveData.Impact.BaseMetricV2.CvssV2.AvailabilityImpact - lousisv2.Version = CveData.Impact.BaseMetricV2.CvssV2.Version - lousisv2.ConfidentialityImpact = CveData.Impact.BaseMetricV2.CvssV2.ConfidentialityImpact - lousisv2.IntegrityImpact = CveData.Impact.BaseMetricV2.CvssV2.IntegrityImpact - lousisv2.AccessVector = CveData.Impact.BaseMetricV2.CvssV2.AccessVector - lousisv2.UserInteractionRequired = CveData.Impact.BaseMetricV2.UserInteractionRequired - lousisv2.Severity = CveData.Impact.BaseMetricV2.Severity - lousisv2.ObtainUserPrivilege = CveData.Impact.BaseMetricV2.ObtainUserPrivilege - lousisv2.ObtainAllPrivilege = CveData.Impact.BaseMetricV2.ObtainAllPrivilege - lousisv2.ObtainOtherPrivilege = CveData.Impact.BaseMetricV2.ObtainOtherPrivilege - lousisv2.ImpactScore = CveData.Impact.BaseMetricV2.ImpactScore - lousisv2.ExploitabilityScore = CveData.Impact.BaseMetricV2.ExploitabilityScore - lousisv2.CveLevel = OpenEulerScoreProc(CveData.Impact.BaseMetricV2.CvssV2.BaseScore) - if lousistnumv2, err := o.Insert(&lousisv2); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score_v2 success, "+ - "lousistnumv2:", lousistnumv2, ", cveNum: ", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score_v2 failed,"+ - " lousisv2:", lousisv2, ", err: ", err) - o.Rollback() + + if err := handleImpact(o, CveData.Impact, num, ou.Ids); err != nil { return 0, err } @@ -702,130 +579,8 @@ func CreateOriginCve(CveData common.CveOriginData, ou *OriginUpstream, od *Origi } } } - osi.CveId = num - losi := OriginUpstreamImpact{CveId: num} - errxx := o.Read(&losi, "CveId") - if errxx == orm.ErrNoRows || errxx == orm.ErrMissPK { - logs.Info("CreateOriginCve, cve_origin_upstream_impact, Data does not exist, cveId: ", num) - } else { - var losis []OriginUpstreamImpactScore - losisNum, err := o.Raw("select * from cve_origin_upstream_impact_score where impact_id = ?", - losi.ImpactId).QueryRows(&losis) - if err != nil { - logs.Info("CreateOriginCve, cve_origin_upstream_impact_score, Data does not exist,"+ - " losi.ImpactId: ", losi.ImpactId, ", err: ", err, ",losisNum: ", losisNum) - } else { - for _, sis := range losis { - if sis.CvssV3 == 1 && sis.BaseMetricV3 == 1 { - lousisv3 := OriginUpstreamImpactScoreV3{ScoreId: sis.ScoreId} - o.Delete(&lousisv3, "ScoreId") - } - if sis.CvssV2 == 1 && sis.BaseMetricV2 == 1 { - lousisv2 := OriginUpstreamImpactScoreV2{ScoreId: sis.ScoreId} - o.Delete(&lousisv2, "ScoreId") - } - } - losisx := OriginUpstreamImpactScore{ImpactId: losi.ImpactId} - o.Delete(&losisx, "ImpactId") - } - losix := OriginUpstreamImpact{CveId: num} - o.Delete(&losix, "CveId") - } - if losinum, err := o.Insert(osi); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact success, "+ - "lousnum:", losinum, ", cveNum", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact failed, "+ - "Lnode:", osi, ", err: ", err) - o.Rollback() - return 0, err - } - var lousist OriginUpstreamImpactScore - lousist.ImpactId = osi.ImpactId - lousist.BaseMetricV3 = 1 - lousist.BaseMetricV2 = 0 - lousist.CvssV3 = 1 - lousist.CvssV2 = 0 - lousist.Status = 1 - if lousistnum, err := o.Insert(&lousist); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score success, "+ - "lousistnum:", lousistnum, ", cveNum", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score failed, "+ - "lousist:", lousist, ", err: ", err) - o.Rollback() - return 0, err - } - var lousisv3 OriginUpstreamImpactScoreV3 - lousisv3.ScoreId = lousist.ScoreId - lousisv3.BaseScore = CveData.Impact.BaseMetricV3.CvssV3.BaseScore - lousisv3.VectorString = CveData.Impact.BaseMetricV3.CvssV3.VectorString - lousisv3.AttackComplexity = CveData.Impact.BaseMetricV3.CvssV3.AttackComplexity - lousisv3.AttackVector = CveData.Impact.BaseMetricV3.CvssV3.AttackVector - lousisv3.AvailabilityImpact = CveData.Impact.BaseMetricV3.CvssV3.AvailabilityImpact - lousisv3.BaseSeverity = CveData.Impact.BaseMetricV3.CvssV3.BaseSeverity - lousisv3.UserInteraction = CveData.Impact.BaseMetricV3.CvssV3.UserInteraction - lousisv3.PrivilegesRequired = CveData.Impact.BaseMetricV3.CvssV3.PrivilegesRequired - lousisv3.Version = CveData.Impact.BaseMetricV3.CvssV3.Version - lousisv3.ConfidentialityImpact = CveData.Impact.BaseMetricV3.CvssV3.ConfidentialityImpact - lousisv3.IntegrityImpact = CveData.Impact.BaseMetricV3.CvssV3.IntegrityImpact - lousisv3.Scope = CveData.Impact.BaseMetricV3.CvssV3.Scope - lousisv3.ImpactScore = CveData.Impact.BaseMetricV3.ImpactScore - lousisv3.ExploitabilityScore = CveData.Impact.BaseMetricV3.ExploitabilityScore - lousisv3.CveLevel = OpenEulerScoreProc(CveData.Impact.BaseMetricV3.CvssV3.BaseScore) - if lousistnumv3, err := o.Insert(&lousisv3); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score_v3 success, "+ - "lousistnumv3:", lousistnumv3, ", cveNum", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score_v3 failed, "+ - "lousisv3:", lousisv3, ", err: ", err) - o.Rollback() - return 0, err - } - var lousistv2 OriginUpstreamImpactScore - lousistv2.ImpactId = osi.ImpactId - lousistv2.BaseMetricV3 = 0 - lousistv2.BaseMetricV2 = 1 - lousistv2.CvssV3 = 0 - lousistv2.CvssV2 = 1 - lousistv2.Status = 1 - if lousistnum, err := o.Insert(&lousistv2); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score success, "+ - "lousistnum:", lousistnum, ", cveNum: ", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score failed, "+ - "lousistv2:", lousistv2, ", err: ", err) - o.Rollback() - return 0, err - } - var lousisv2 OriginUpstreamImpactScoreV2 - lousisv2.ScoreId = lousistv2.ScoreId - lousisv2.BaseScore = CveData.Impact.BaseMetricV2.CvssV2.BaseScore - lousisv2.AcInsufInfo = CveData.Impact.BaseMetricV2.AcInsufInfo - lousisv2.BaseScore = CveData.Impact.BaseMetricV2.CvssV2.BaseScore - lousisv2.VectorString = CveData.Impact.BaseMetricV2.CvssV2.VectorString - lousisv2.AccessComplexity = CveData.Impact.BaseMetricV2.CvssV2.AccessComplexity - lousisv2.Authentication = CveData.Impact.BaseMetricV2.CvssV2.Authentication - lousisv2.AvailabilityImpact = CveData.Impact.BaseMetricV2.CvssV2.AvailabilityImpact - lousisv2.Version = CveData.Impact.BaseMetricV2.CvssV2.Version - lousisv2.ConfidentialityImpact = CveData.Impact.BaseMetricV2.CvssV2.ConfidentialityImpact - lousisv2.IntegrityImpact = CveData.Impact.BaseMetricV2.CvssV2.IntegrityImpact - lousisv2.AccessVector = CveData.Impact.BaseMetricV2.CvssV2.AccessVector - lousisv2.UserInteractionRequired = CveData.Impact.BaseMetricV2.UserInteractionRequired - lousisv2.Severity = CveData.Impact.BaseMetricV2.Severity - lousisv2.ObtainUserPrivilege = CveData.Impact.BaseMetricV2.ObtainUserPrivilege - lousisv2.ObtainAllPrivilege = CveData.Impact.BaseMetricV2.ObtainAllPrivilege - lousisv2.ObtainOtherPrivilege = CveData.Impact.BaseMetricV2.ObtainOtherPrivilege - lousisv2.ImpactScore = CveData.Impact.BaseMetricV2.ImpactScore - lousisv2.ExploitabilityScore = CveData.Impact.BaseMetricV2.ExploitabilityScore - lousisv2.CveLevel = OpenEulerScoreProc(CveData.Impact.BaseMetricV2.CvssV2.BaseScore) - if lousistnumv2, err := o.Insert(&lousisv2); err == nil { - logs.Info("CreateOriginCve, insert cve_origin_upstream_impact_score_v2 success, "+ - "lousistnumv2:", lousistnumv2, ", cveNum: ", ou.Ids) - } else { - logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score_v2 failed, "+ - "lousisv2:", lousisv2, ", err: ", err) - o.Rollback() + + if err := handleImpact(o, CveData.Impact, ouse.CveId, ou.Ids); err != nil { return 0, err } @@ -1001,6 +756,100 @@ func CreateOriginCve(CveData common.CveOriginData, ou *OriginUpstream, od *Origi return 0, nil } +func clearOldImpact(o orm.Ormer, cveId int64, oldImpact OriginUpstreamImpact) { + var losis []OriginUpstreamImpactScore + losisNum, err := o.Raw("select * from cve_origin_upstream_impact_score where impact_id = ?", + oldImpact.ImpactId).QueryRows(&losis) + if err != nil { + logs.Info("CreateOriginCve, cve_origin_upstream_impact_score, Data does not exist,"+ + " losi.ImpactId: ", oldImpact.ImpactId, ", err: ", err, ",losisNum: ", losisNum) + } else { + for _, sis := range losis { + if sis.CvssV3 == 1 && sis.BaseMetricV3 == 1 { + lousisv3 := OriginUpstreamImpactScoreV3{ScoreId: sis.ScoreId} + o.Delete(&lousisv3, "ScoreId") + } + if sis.CvssV2 == 1 && sis.BaseMetricV2 == 1 { + lousisv2 := OriginUpstreamImpactScoreV2{ScoreId: sis.ScoreId} + o.Delete(&lousisv2, "ScoreId") + } + } + losisx := OriginUpstreamImpactScore{ImpactId: oldImpact.ImpactId} + o.Delete(&losisx, "ImpactId") + } + losix := OriginUpstreamImpact{CveId: cveId} + o.Delete(&losix, "CveId") +} + +func saveNewImpact(o orm.Ormer, cveId int64, source string, impact common.CveImpact) error { + osi := new(OriginUpstreamImpact) + osi.CveId = cveId + osi.Source = source + if _, err := o.Insert(osi); err != nil { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact failed,"+" Lnode:", osi, ", err: ", err) + o.Rollback() + return err + } + + var lousist OriginUpstreamImpactScore + lousist.ImpactId = osi.ImpactId + lousist.BaseMetricV3 = 1 + lousist.BaseMetricV2 = 0 + lousist.CvssV3 = 1 + lousist.CvssV2 = 0 + lousist.Status = 1 + if _, err := o.Insert(&lousist); err != nil { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score failed ", err) + o.Rollback() + return err + } + var lousisv3 OriginUpstreamImpactScoreV3 + lousisv3.ScoreId = lousist.ScoreId + lousisv3.BaseScore = impact.BaseMetricV3.CvssV3.BaseScore + lousisv3.VectorString = impact.BaseMetricV3.CvssV3.VectorString + lousisv3.AttackComplexity = impact.BaseMetricV3.CvssV3.AttackComplexity + lousisv3.AttackVector = impact.BaseMetricV3.CvssV3.AttackVector + lousisv3.AvailabilityImpact = impact.BaseMetricV3.CvssV3.AvailabilityImpact + lousisv3.BaseSeverity = impact.BaseMetricV3.CvssV3.BaseSeverity + lousisv3.UserInteraction = impact.BaseMetricV3.CvssV3.UserInteraction + lousisv3.PrivilegesRequired = impact.BaseMetricV3.CvssV3.PrivilegesRequired + lousisv3.Version = impact.BaseMetricV3.CvssV3.Version + lousisv3.ConfidentialityImpact = impact.BaseMetricV3.CvssV3.ConfidentialityImpact + lousisv3.IntegrityImpact = impact.BaseMetricV3.CvssV3.IntegrityImpact + lousisv3.Scope = impact.BaseMetricV3.CvssV3.Scope + lousisv3.ImpactScore = impact.BaseMetricV3.ImpactScore + lousisv3.ExploitabilityScore = impact.BaseMetricV3.ExploitabilityScore + lousisv3.CveLevel = OpenEulerScoreProc(impact.BaseMetricV3.CvssV3.BaseScore) + if _, err := o.Insert(&lousisv3); err != nil { + logs.Error("CreateOriginCve, insert cve_origin_upstream_impact_score_v3 failed", err) + o.Rollback() + return err + } + + return nil +} + +func handleImpact(o orm.Ormer, newImpact map[string]common.CveImpact, cveId int64, cveNum string) error { + var impacts []OriginUpstreamImpact + _, err := o.QueryTable(&OriginUpstreamImpact{}).Filter("cve_id", cveId).All(&impacts) + if err != nil { + return err + } + + for _, v := range impacts { + clearOldImpact(o, cveId, v) + } + + for source, v := range newImpact { + if err = saveNewImpact(o, cveId, source, v); err != nil { + logs.Error("saveNewImpact of %s failed:%s", cveNum, err.Error()) + return err + } + } + + return nil +} + func resetOriginPatch(cveData *common.CveOriginData, o orm.Ormer, oid string) error { if cveData == nil { return nil diff --git a/cve-vulner-manager/routers/commentsRouter_controllers.go b/cve-vulner-manager/routers/commentsRouter_controllers.go index b1b9d076f0a524eed7a3ac2c17aee87600f85b79..4801192b5b4d4b02469d7666c79832e5a74eda7c 100644 --- a/cve-vulner-manager/routers/commentsRouter_controllers.go +++ b/cve-vulner-manager/routers/commentsRouter_controllers.go @@ -94,24 +94,6 @@ func initComment() { MethodParams: param.Make(), Filters: nil, Params: nil}) - - beego.GlobalControllerRouter["cvevulner/controllers:FileController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:FileController"], - beego.ControllerComments{ - Method: "TriggerCveData", - Router: "/triggerCveData", - AllowHTTPMethods: []string{"get"}, - MethodParams: param.Make(), - Filters: nil, - Params: nil}) - beego.GlobalControllerRouter["cvevulner/controllers:FileController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:FileController"], - beego.ControllerComments{ - Method: "TriggerUnaffectedCve", - Router: "/triggerUnaffectedCve", - AllowHTTPMethods: []string{"get"}, - MethodParams: param.Make(), - Filters: nil, - Params: nil}) - beego.GlobalControllerRouter["cvevulner/controllers:GaussCveController"] = append(beego.GlobalControllerRouter["cvevulner/controllers:GaussCveController"], beego.ControllerComments{ Method: "Get", diff --git a/cve-vulner-manager/taskhandler/common.go b/cve-vulner-manager/taskhandler/common.go index 7a9dab2cd60506347266d1b5d98556608c7fbeed..e8e532b9700b9559fafb3808e56abdfc56b4ce05 100644 --- a/cve-vulner-manager/taskhandler/common.go +++ b/cve-vulner-manager/taskhandler/common.go @@ -931,6 +931,9 @@ func CreateIssueBody(accessToken, owner, path, assignee string, if err != nil { nvdScore = 0.0 } + if nvdScore == 0 { + score = "N/A" + } cveAnalysis := "" if its.CveAnalysis != "" && len(its.CveAnalysis) > 1 { diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index 7aac53b42246d7dd691bd9de4594d1a2c3dbe4c9..270a972993bffe160b59595fb568c3d64f20064c 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -447,11 +447,11 @@ func UpdateIssueToGit(accessToken, owner, path string, ",its: ", its, ", owner: ", owner, ",path: ", path) if len(ignoreStatus) == 0 { - if its.Status == 4 || its.Status == 5 { + if its.Status == models.IssueStatusRejected { logs.Error("UpdateIssueToGit, "+ - "The current issue has been suspended/rejected and will not be processed, its: ", its.IssueNum) + "The current issue has been rejected and will not be processed, its: ", its.IssueNum) models.UpdateIssueStatus(cve, 2) - return "", errors.New("The current issue has been suspended/rejected and will not be processed") + return "", errors.New("the current issue has been rejected and will not be processed") } } owner, accessToken = common.GetOwnerAndToken(cve.CveNum, cve.OrganizationID) diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index bbe06a7a7546dd118b0f7b6f7cf03e42424de73b..8136b222ccf23bb81d9ce286b2fd6adb6b1c8dc9 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -1,15 +1,11 @@ package taskhandler import ( - "encoding/base64" "encoding/json" "errors" - "fmt" "io/ioutil" "net/http" "net/url" - "os" - "path/filepath" "regexp" "strconv" "strings" @@ -29,8 +25,6 @@ import ( "github.com/astaxie/beego/logs" ) -var GetCveDetailUrl = "https://openeuler.org/api-cve/cve-security-notice-server/cvedatabase/getByCveId?cveId=%s" - var ( pkgLock sync.Mutex lockx sync.Mutex @@ -384,7 +378,7 @@ func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum //CveRes.PackName = pkList[0] CveRes.RepoName = pkList[0] CveRes.CveUrl = cveRef + cveData.CveNum - CveRes.CveLevel = cveScV3.CveLevel + CveRes.CveLevel = models.OpenEulerScoreProc(cveScV3.BaseScore) CveRes.OrganizationID = organizationID CveRes.FirstPerTime = cveData.FirstPerTime CveRes.FirstGetTime = cveData.FirstGetTime @@ -656,7 +650,7 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef, repoNme string, vul.RepoName = pkList[0] vul.PackName = repoNme vul.CveUrl = cveRef + cveData.CveNum - vul.CveLevel = cveScV3.CveLevel + vul.CveLevel = models.OpenEulerScoreProc(cveScV3.BaseScore) if common.GetCveSource(cveData.CveNum) == 1 || common.GetCveSourceRecord(cveData.CveNum) == 1 { vul.DataSource = 5 } else { @@ -665,6 +659,7 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef, repoNme string, vul.IsExport = 0 vul.OrganizationID = organizationID vul.AffectUpdate = cveData.AffectUpdate + vul.PackageTime = cveData.GetPackageTime(repoNme) var sec models.SecurityNotice sec.CveNum = cveData.CveNum @@ -851,27 +846,11 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve description information is empty", 3) return false, errors.New("数据错误，暂时不处理") } - cveImpact, ok := models.QueryCveImpact(cveData.CveId) - if !ok { - logs.Error("GenCveVuler, QueryCveImpact, Data query failed, data: ", cveData) - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) - UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 1", 3) - return false, errors.New("数据错误，暂时不处理") - } + cveImpact, _ := models.QueryCveImpact(cveData.CveId, "nvd") cveScV2 := getScoreV2(cveImpact.ImpactId) cveScV3 := getScoreV3(cveImpact.ImpactId) - if cveScV3.V3Id == 0 && cveScV2.V2Id == 0 { - logs.Error("GenCveVuler, can not get score of data: ", cveData) - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) - UpdateOriginUpstreamRecord(cveData.Ids, cveData.CveNum, "cve score information is empty 3", 3) - return false, errors.New("数据错误，暂时不处理") - } - scopeType := "v3" - if cveScV3.BaseScore == 0 && cveScV2.BaseScore > 0 { - scopeType = "v2" - } packNameMap := map[string]string{} packNameList := []string{} @@ -2579,18 +2558,6 @@ func IsProductReleased(cveNumber, packageName, branch string) (bool, string) { return false, "" } -// IsProductFixed checks if a product (branch) is fixed for a given CVE number and package name. -func IsProductFixed(cveNumber, packageName, branch string) bool { - products := GetCveProduct(cveNumber, packageName) - for _, v := range products { - if v.IsFixed() && v.ProductName == branch { - return true - } - } - - return false -} - // GetCveProduct retrieves a list of CVE products based on the provided CVE number and package name. func GetCveProduct(cveNumber, packageName string) []models.CveProduct { var detail models.RespCveProduct @@ -2672,91 +2639,6 @@ func GenerateExcelTask() error { return nil } -// GenerateExcelTrigger generate cve&security notice excel file by pr merge and influence package release. -func GenerateExcelTrigger(wgCrvf *sync.WaitGroup, fileName, startTime, fileCode, affectBranch, - csvDownPath, dir string, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, cves []string) UpdateInfoXml { - defer wgCrvf.Done() - logs.Info(affectBranch, ", GenerateExcelTrigger start...") - var updateInfoXml UpdateInfoXml - xmlFileName := beego.AppConfig.String("xml::updateinfo_path") - xmfilePath := filepath.Join(dir, affectBranch+"_"+xmlFileName) - var cvexml []CveXml - var dpdates Updates - securityNotice := make(map[string][]SecurityNoticeXml) - packRpmx := make(map[string][]PackRpm) - ReadXml(xmfilePath, &dpdates) - fileName = filepath.Join(dir, fileName) - localPath := filepath.Join(dir, affectBranch+"_release-package.CSV") - err := downloadPackageFile(localPath, csvDownPath, affectBranch) - if err != nil { - logs.Error(err) - return updateInfoXml - } - pkgList, err := ExtractPackageData(localPath) - if err != nil { - logs.Error(err) - } else { - su := time.Now().Format("2006-01-02") - snPrefix := "openEuler-" + su - snSuffix := int64(1001) - err = GenerateCveExcelByTrigger(affectBranch, fileName, snPrefix, startTime, - snSuffix, true, pkgList, cvrfFileList, - componentMap, cvfrFileMap, &cvexml, &dpdates, - securityNotice, packRpmx, cves) - if err != nil { - logs.Error(err) - } - } - updateInfoXml = UpdateInfoXml{XmfilePath: xmfilePath, Cvexml: cvexml, Dpdates: &dpdates, - SecurityNotice: securityNotice, PackRpmx: packRpmx, FileName: fileName, AffectBranch: affectBranch} - logs.Info(affectBranch, ", GenerateExcelTrigger: end") - return updateInfoXml -} - -func downloadPackageFile(localPath, url, affectBranch string) error { - logs.Info("The csv file currently being downloaded is: ", url) - pkgLock.Lock() - err := GiteeDownCsv(localPath, affectBranch) - pkgLock.Unlock() - return err -} - -func GiteeDownCsv(localPath, branch string) error { - gitUrl := fmt.Sprintf( - "https://gitee.com/api/v5/repos/%s/%s/contents/%s?access_token=%s", - "openeuler_latest_rpms", "obs_pkg_rpms_20231109", fmt.Sprintf("latest_rpm/%v.csv", branch), os.Getenv("GITEE_TOKEN"), - ) - resp, err := http.Get(gitUrl) - if err != nil { - logs.Error("get error, url:", gitUrl, "error: ", err) - return err - } - defer resp.Body.Close() - body, err := ioutil.ReadAll(resp.Body) - if err != nil || body == nil { - return err - } - var contents map[string]interface{} - if err = json.Unmarshal(body, &contents); err != nil { - return err - } - f, ferr := os.Create(localPath) - if ferr != nil { - return ferr - } - - if contents["type"].(string) == "file" && contents["encoding"].(string) == "base64" { - data, baseErr := base64.StdEncoding.DecodeString(contents["content"].(string)) - if baseErr == nil { - f.Write(data) - } - } else { - f.WriteString(contents["content"].(string)) - } - return nil -} - func CvePlanCloseTime(startTime time.Time, cveLevel string, isLocalFlag, isTparam bool) string { endLocal := "" switch cveLevel { diff --git a/cve-vulner-manager/taskhandler/cvrf.go b/cve-vulner-manager/taskhandler/cvrf.go index 862002450921274de3930293c9f2ecf2e17c2e5d..eecefdad666124c723849c3322ec9bb89613cd24 100644 --- a/cve-vulner-manager/taskhandler/cvrf.go +++ b/cve-vulner-manager/taskhandler/cvrf.go @@ -1,24 +1,17 @@ package taskhandler import ( - "bytes" - "compress/zlib" "encoding/xml" - "errors" "fmt" - "io" "io/ioutil" - "net/http" "os" "path/filepath" - "sort" "strconv" "strings" "cvevulner/common" "cvevulner/models" - "github.com/astaxie/beego" "github.com/astaxie/beego/logs" ) @@ -299,126 +292,6 @@ type UnaffectVulnerability struct { Remediations *UnRemediations `xml:"Remediations,omitempty"` } -func UnaffectReadCvrfXml(filePath string, unaffectCvrfSa *UnaffectCvrfSa) error { - if filePath == "" || len(filePath) == 0 { - logs.Error("read err: ", filePath) - return errors.New("file does not exist") - } - fisExist, ferr := PathExists(filePath) - if !fisExist { - logs.Error(ferr) - return ferr - } - fd, err := os.Open(filePath) - if err != nil { - logs.Error("open file err : ", err, ",filePath: ", filePath) - return err - } - defer fd.Close() - fileContent, err := ioutil.ReadAll(fd) - if err != nil { - logs.Error("read file err : ", err, ", filePath: ", filePath) - return err - } - err = xml.Unmarshal(fileContent, unaffectCvrfSa) - if err != nil { - logs.Error("unmarshal err : ", err, ", fileContent: ", fileContent) - return err - } - return nil -} - -func WriteUnaffectCvrfXml(filePath string, unaffectCvrfsa *UnaffectCvrfSa) { - os.Remove(filePath) - xmlOutPut, outPutErr := xml.MarshalIndent(unaffectCvrfsa, "", " ") - if outPutErr == nil { - headerBytes := []byte(xml.Header) - xmlOutPutData := append(headerBytes, xmlOutPut...) - ioutil.WriteFile(filePath, xmlOutPutData, os.ModePerm) - } else { - logs.Error(outPutErr) - } -} - -func ReadCvrfXml(filePath string, cvrf *CvrfSa) error { - if filePath == "" || len(filePath) == 0 { - logs.Error("read err: ", filePath) - return errors.New("file does not exist") - } - fisExist, ferr := PathExists(filePath) - if !fisExist { - logs.Error(ferr) - return ferr - } - fd, err := os.Open(filePath) - if err != nil { - logs.Error("open file err : ", err, ",filePath: ", filePath) - return err - } - defer fd.Close() - fileContent, err := ioutil.ReadAll(fd) - if err != nil { - logs.Error("read file err : ", err, ", filePath: ", filePath) - return err - } - err = xml.Unmarshal(fileContent, cvrf) - if err != nil { - logs.Error("unmarshal err : ", err, ", fileContent: ", fileContent) - return err - } - return nil -} - -func WriteCvrfXml(filePath string, cvrfsa *CvrfSa) { - os.Remove(filePath) - CvrfSaTitle(cvrfsa) - xmlOutPut, outPutErr := xml.MarshalIndent(cvrfsa, "", " ") - if outPutErr == nil { - headerBytes := []byte(xml.Header) - xmlOutPutData := append(headerBytes, xmlOutPut...) - ioutil.WriteFile(filePath, xmlOutPutData, os.ModePerm) - } else { - logs.Error(outPutErr) - } -} - -func CvrfSaTitle(cvrfsa *CvrfSa) { - var title string - var branch = make([]string, 0) - for _, v := range cvrfsa.ProductTree.OpenEulerBranch { - if strings.EqualFold(v.Name, "openEuler") { - for _, br := range v.FullProductName { - branch = append(branch, br.FullProductName) - } - } - } - if len(branch) > 0 { - sort.Strings(branch) - old := cvrfsa.DocumentTitle.DocumentTitle - index := strings.Index(old, "openEuler") - title = old[:index] - if len(branch) == 1 { - title = title + branch[0] - } else if len(branch) == 2 { - title = title + strings.Join(branch, " and ") - } else { - title = title + strings.Join(branch[:len(branch)-1], ",") + " and " + branch[len(branch)-1] - } - cvrfsa.DocumentTitle.DocumentTitle = title - for k := range cvrfsa.DocumentNotes.Note { - if strings.EqualFold(cvrfsa.DocumentNotes.Note[k].Title, "Summary") { - cvrfsa.DocumentNotes.Note[k].Note = title + "." - } - if strings.EqualFold(cvrfsa.DocumentNotes.Note[k].Title, "Topic") { - topic := strings.Split(cvrfsa.DocumentNotes.Note[k].Note, "\r\n\r\n") - if len(topic) == 2 { - cvrfsa.DocumentNotes.Note[k].Note = title + "." + "\r\n\r\n" + topic[1] - } - } - } - } -} - func BuildUnaffectVulnerabilitySet(unaffectCvrfsa *UnaffectCvrfSa, v models.ExcelExport, affectBranch string, componentMap map[string]ComponentInfo) { vulnerability := make([]UnaffectVulnerability, 0) @@ -542,8 +415,8 @@ func BuildUnaffVulnerability(vlLenth int, v models.ExcelExport, vulnerability.ProductStatuses = &productStatuses var cVSSScoreSets CVSSScoreSets var scoreSet ScoreSet - scoreSet.BaseScore = fmt.Sprintf("%.1f", v.OpenEulerScore) - scoreSet.Vector = v.OvectorVule + scoreSet.BaseScore = fmt.Sprintf("%.1f", v.OpeneulerScore) + scoreSet.Vector = v.OpeneulerVector cVSSScoreSets.ScoreSet = &scoreSet vulnerability.CvssScoreSets = &cVSSScoreSets var remediations UnRemediations @@ -561,857 +434,6 @@ func BuildUnaffVulnerability(vlLenth int, v models.ExcelExport, return vulnerabilitySlice } -func BranchExist(affectBranch string, cvrfFileList map[string][]string) []string { - brancsList, keyOk := cvrfFileList[BRANCHSKEY] - if !keyOk { - localBran := make([]string, 0) - localBran = append(localBran, affectBranch) - cvrfFileList[BRANCHSKEY] = localBran - } else { - brlFlag := false - for _, brl := range brancsList { - if brl == affectBranch { - brlFlag = true - break - } - } - if !brlFlag { - brancsList = append(brancsList, affectBranch) - cvrfFileList[BRANCHSKEY] = brancsList - } - } - brancsListx, _ := cvrfFileList[BRANCHSKEY] - if len(brancsListx) > 1 { - sort.Strings(brancsListx) - } - return brancsListx -} - -func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, - introduction, topic, affectBranch string, - componentMap map[string]ComponentInfo, - branchList []string, branchFlag int) { - documentNotes := cvrfsa.DocumentNotes - if documentNotes == nil || cvrfsa.DocumentNotes.Note == nil || - len(cvrfsa.DocumentNotes.Note) == 0 { - var documentNotesx DocumentNotes - note := make([]Note, 0) - var noteSynopsis Note - noteSynopsis.Title = "Synopsis" - noteSynopsis.Type = "General" - noteSynopsis.Ordinal = "1" - noteSynopsis.XmlLang = "en" - noteSynopsis.Note = v.Summary - note = append(note, noteSynopsis) - var noteSummary Note - noteSummary.Title = "Summary" - noteSummary.Type = "General" - noteSummary.Ordinal = "2" - noteSummary.XmlLang = "en" - noteSummary.Note = introduction - note = append(note, noteSummary) - var noteDescription Note - noteDescription.Title = "Description" - noteDescription.Type = "General" - noteDescription.Ordinal = "3" - noteDescription.XmlLang = "en" - descriptionStr := strings.ReplaceAll(v.Description, "\n\n", "\r\n\r\n") - descriptionStr = XmlSpecCharHand(descriptionStr) - noteDescription.Note = descriptionStr - note = append(note, noteDescription) - var noteTopic Note - noteTopic.Title = "Topic" - noteTopic.Type = "General" - noteTopic.Ordinal = "4" - noteTopic.XmlLang = "en" - topic := strings.ReplaceAll(topic, "\n\n", "\r\n\r\n") - topic = XmlSpecCharHand(topic) - noteTopic.Note = topic - note = append(note, noteTopic) - var noteSeverity Note - noteSeverity.Title = "Severity" - noteSeverity.Type = "General" - noteSeverity.Ordinal = "5" - noteSeverity.XmlLang = "en" - cveLevel := models.OpenEulerScoreProc(v.OpenEulerScore) - noteSeverity.Note = cveLevel - note = append(note, noteSeverity) - var noteComponent Note - noteComponent.Title = "Affected Component" - noteComponent.Type = "General" - noteComponent.Ordinal = "6" - noteComponent.XmlLang = "en" - noteComponent.Note = v.InfluenceComponent - note = append(note, noteComponent) - documentNotesx.Note = note - cvrfsa.DocumentNotes = &documentNotesx - } else { - branchCount := len(branchList) - note := cvrfsa.DocumentNotes.Note - ownedComponent := "" - if len(v.InfluenceComponent) > 1 { - ownedComponent = v.InfluenceComponent - } else { - ownedComponent = v.OwnedComponent - } - componentInfo := componentMap[ownedComponent] - notex := make([]Note, 0) - for _, te := range note { - if te.Title == "Description" { - descriptionStr := strings.ReplaceAll(v.Description, "\n\n", "\r\n\r\n") - descriptionStr = XmlSpecCharHand(descriptionStr) - dSplit := strings.Split(descriptionStr, "Security Fix(es):") - if len(dSplit) > 1 { - if !strings.Contains(te.Note, dSplit[0]) { - te.Note = dSplit[0] + te.Note - } - if !strings.Contains(te.Note, dSplit[1]) { - te.Note += dSplit[1] - } - } - te.Note = te.Note - } - if te.Title == "Topic" { - vcn := "" - if branchFlag == 1 { - vcn = strings.Join(componentInfo.CveNum, ";\n") - theme, err := models.GetCanExportTheme(vcn, v.InfluenceComponent, affectBranch) - if err == nil && len(theme) > 1 { - theme = strings.ReplaceAll(theme, "\n\n", "\r\n\r\n") - theme = XmlSpecCharHand(theme) - if branchCount <= 1 { - te.Note = theme - } else if branchCount == 2 { - te.Note = strings.ReplaceAll(theme, affectBranch, strings.Join(branchList, " and ")) - } else { - reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] - te.Note = strings.ReplaceAll(theme, affectBranch, reBanch) - } - } - } else { - vcn = strings.Join(componentInfo.CveNumMap[affectBranch].CveNumSlice, ";\n") - theme, err := models.GetCanExportTheme(vcn, v.InfluenceComponent, affectBranch) - if err == nil && len(theme) > 1 { - theme = strings.ReplaceAll(theme, "\n\n", "\r\n\r\n") - theme = XmlSpecCharHand(theme) - te.Note = theme - } - } - } - if te.Title == "Summary" { - if branchFlag == 1 { - if branchCount <= 1 { - te.Note = v.Introduction - } else if branchCount == 2 { - te.Note = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) - } else { - reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] - te.Note = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) - } - } else { - te.Note = v.Introduction - } - - } - if te.Title == "Severity" { - var openEulerScoreSlice []float64 - if branchFlag == 1 { - openEulerScoreSlice = componentInfo.OpenEulerScore - } else { - openEulerScoreSlice = componentInfo.CveNumMap[affectBranch].OpenEulerScore - } - - if len(openEulerScoreSlice) > 1 { - sort.Float64s(openEulerScoreSlice) - } - if len(openEulerScoreSlice) > 0 { - cveLevel := models.OpenEulerScoreProc(openEulerScoreSlice[len(openEulerScoreSlice)-1]) - te.Note = cveLevel - } - } - notex = append(notex, te) - } - cvrfsa.DocumentNotes.Note = notex - } -} - -func BuildDocumentTitle(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, - cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, - curDate string, branchList []string, branchFlag int) { - var documt DocumentTitle - introduction := "" - topic := "" - branchCount := len(branchList) - ownedComponent := "" - if len(v.InfluenceComponent) > 1 { - ownedComponent = v.InfluenceComponent - } else { - ownedComponent = v.OwnedComponent - } - componentInfo := componentMap[ownedComponent] - localOpenEulerSANum := "" - if branchFlag == 1 { - localOpenEulerSANum = componentInfo.OpenEulerSANum - if branchCount <= 1 { - introduction = v.Introduction - topic = v.Theme - } else if branchCount == 2 { - introduction = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) - topic = strings.ReplaceAll(v.Theme, affectBranch, strings.Join(branchList, " and ")) - } else { - reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] - introduction = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) - topic = strings.ReplaceAll(v.Theme, affectBranch, reBanch) - } - } else { - introduction = v.Introduction - topic = v.Theme - localOpenEulerSANum = componentInfo.CveNumMap[affectBranch].OpenEulerSANum - } - if len(introduction) > 1 { - documt.DocumentTitle = introduction[:len(introduction)-1] - } else { - documt.DocumentTitle = introduction - } - documt.XmlLang = "en" - cvrfsa.DocumentTitle = &documt - cvrfsa.DocumentType = "Security Advisory" - var documentPublisher DocumentPublisher - documentPublisher.Type = "Vendor" - documentPublisher.ContactDetails = "openeuler-security@openeuler.org" - documentPublisher.IssuingAuthority = "openEuler security committee" - cvrfsa.DocumentPublisher = &documentPublisher - var documentTracking DocumentTracking - var identification Identification - identification.Id = localOpenEulerSANum - documentTracking.Identification = &identification - documentTracking.Status = "Final" - documentTracking.Version = "1.0" - var revisionHistory RevisionHistory - revision := make([]Revision, 0) - if cvrfsa.DocumentTracking != nil && - cvrfsa.DocumentTracking.RevisionHistory != nil && - len(cvrfsa.DocumentTracking.RevisionHistory.Revision) > 0 && - componentInfo.UpdateFlag == 1 { - RevisionSlice := cvrfsa.DocumentTracking.RevisionHistory.Revision - for _, rev := range RevisionSlice { - revision = append(revision, rev) - } - versionValue := float64(len(RevisionSlice))/float64(10) + float64(1) - var revisionx Revision - revisionx.Number = fmt.Sprintf("%.1f", versionValue) - revisionx.Date = v.PublicDate - revisionx.Description = "Update" - revision = append(revision, revisionx) - documentTracking.Version = revisionx.Number - } else { - var revisionx Revision - revisionx.Number = "1.0" - revisionx.Date = v.PublicDate - revisionx.Description = "Initial" - revision = append(revision, revisionx) - documentTracking.InitialReleaseDate = v.PublicDate - } - revisionHistory.Revision = revision - documentTracking.RevisionHistory = &revisionHistory - documentTracking.CurrentReleaseDate = v.PublicDate - var generator Generator - generator.Date = curDate - generator.Engine = "openEuler SA Tool V1.0" - documentTracking.Generator = &generator - cvrfsa.DocumentTracking = &documentTracking - BuilddocumentNotes(cvrfsa, v, introduction, topic, - affectBranch, componentMap, branchList, branchFlag) -} - -func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, - componentMap map[string]ComponentInfo, branchFlag int, affectBranch string) { - componentInfo, comOk := componentMap[v.InfluenceComponent] - localOpenEulerSANum := "" - secLinkConfig := beego.AppConfig.String("reflink::openeuler_web") - localCveNum := []string{} - if branchFlag == 1 { - localOpenEulerSANum = componentInfo.OpenEulerSANum - localCveNum = componentInfo.CveNum - } else { - localOpenEulerSANum = componentInfo.CveNumMap[affectBranch].OpenEulerSANum - localCveNum = componentInfo.CveNumMap[affectBranch].CveNumSlice - } - if comOk { - if cvrfsa.DocumentReferences != nil && len(cvrfsa.DocumentReferences.CveReference) > 0 { - cveReference := cvrfsa.DocumentReferences.CveReference - cveReferencex := make([]CveReference, 0) - for _, cveRef := range cveReference { - if cveRef.Type == "openEuler CVE" { - isExist := false - for _, cUrl := range cveRef.CveUrl { - if strings.Contains(cUrl.Url, v.CveNum) { - isExist = true - } - } - if !isExist { - var cveUrl1 CveUrl - cveUrl1.Url = secLinkConfig + "/en/security/cve/detail/?cveId=" + v.CveNum - cveRef.CveUrl = append(cveRef.CveUrl, cveUrl1) - } - } - if cveRef.Type == "Other" { - isExist := false - for _, cUrl := range cveRef.CveUrl { - if strings.Contains(cUrl.Url, v.CveNum) { - isExist = true - } - } - if !isExist { - var cveUrl2 CveUrl - cveUrl2.Url = "https://nvd.nist.gov/vuln/detail/" + v.CveNum - cveRef.CveUrl = append(cveRef.CveUrl, cveUrl2) - } - } - cveReferencex = append(cveReferencex, cveRef) - } - cvrfsa.DocumentReferences.CveReference = cveReferencex - } else { - var documentReferences DocumentReferences - cveReferenceSlice := make([]CveReference, 0) - var cveReference0 CveReference - cveUrlSlice0 := make([]CveUrl, 0) - var cveUrl0 CveUrl - cveUrl0.Url = secLinkConfig + "/en/security/security-bulletins/detail?id=" + localOpenEulerSANum - cveUrlSlice0 = append(cveUrlSlice0, cveUrl0) - cveReference0.Type = "Self" - cveReference0.CveUrl = cveUrlSlice0 - cveReferenceSlice = append(cveReferenceSlice, cveReference0) - var cveReference1 CveReference - cveUrlSlice1 := make([]CveUrl, 0) - for _, cveNum := range localCveNum { - var cveUrl1 CveUrl - cveUrl1.Url = secLinkConfig + "/en/security/cve/detail/?cveId=" + cveNum - cveUrlSlice1 = append(cveUrlSlice1, cveUrl1) - } - cveReference1.Type = "openEuler CVE" - cveReference1.CveUrl = cveUrlSlice1 - cveReferenceSlice = append(cveReferenceSlice, cveReference1) - var cveReference2 CveReference - cveUrlSlice2 := make([]CveUrl, 0) - for _, cveNum := range localCveNum { - var cveUrl2 CveUrl - cveUrl2.Url = "https://nvd.nist.gov/vuln/detail/" + cveNum - cveUrlSlice2 = append(cveUrlSlice2, cveUrl2) - } - cveReference2.Type = "Other" - cveReference2.CveUrl = cveUrlSlice2 - cveReferenceSlice = append(cveReferenceSlice, cveReference2) - documentReferences.CveReference = cveReferenceSlice - cvrfsa.DocumentReferences = &documentReferences - } - } -} - -func BuildProductTree(cvrfsa *CvrfSa, v models.ExcelExport, pkg []models.Package, - affectBranch string, branchFlag int) { - if cvrfsa.ProductTree != nil && len(cvrfsa.ProductTree.OpenEulerBranch) > 0 && branchFlag == 1 { - affectBranchListx := strings.Split(affectBranch, "-") - cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", - affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) - openEulerBranchx := make([]OpenEulerBranch, 0) - for _, opBranch := range cvrfsa.ProductTree.OpenEulerBranch { - if opBranch.Type == "Product Name" && strings.Contains(affectBranch, opBranch.Name) { - fpFlag := false - for _, fp := range opBranch.FullProductName { - if fp.FullProductName == affectBranch { - fpFlag = true - break - } - } - if !fpFlag { - var fullProductName0 FullProductName - fullProductName0.FullProductName = affectBranch - fullProductName0.ProductId = affectBranch - fullProductName0.Cpe = cpe - opBranch.FullProductName = append(opBranch.FullProductName, fullProductName0) - } - } - if opBranch.Type == "Package Arch" { - fullProductNameSliceaarch64 := make([]FullProductName, 0) - fullProductNameSlicenoarch := make([]FullProductName, 0) - fullProductNameSlicesrc := make([]FullProductName, 0) - fullProductNameSlicex86_64 := make([]FullProductName, 0) - if len(pkg) > 0 { - for _, pk := range pkg { - if opBranch.Name == "aarch64" { - isExist := false - for _, fp := range opBranch.FullProductName { - if fp.FullProductName == pk.PackName && fp.Cpe == cpe { - isExist = true - break - } - } - if !isExist && strings.Contains(pk.PackName, ".aarch64.") { - var fullProductNameaarch64 FullProductName - fullProductNameaarch64.FullProductName = pk.PackName - fullProductNameaarch64.Cpe = cpe - if p, ok := checkRpm(pk.PackName); ok { - fullProductNameaarch64.ProductId = p - } - fullProductNameSliceaarch64 = append(fullProductNameSliceaarch64, fullProductNameaarch64) - } - } else if opBranch.Name == "noarch" { - isExist := false - for _, fp := range opBranch.FullProductName { - if fp.FullProductName == pk.PackName && fp.Cpe == cpe { - isExist = true - break - } - } - if !isExist && strings.Contains(pk.PackName, ".noarch.") { - var fullProductNamenoarch FullProductName - fullProductNamenoarch.FullProductName = pk.PackName - fullProductNamenoarch.Cpe = cpe - if p, ok := checkRpm(pk.PackName); ok { - fullProductNamenoarch.ProductId = p - } - fullProductNameSlicenoarch = append(fullProductNameSlicenoarch, fullProductNamenoarch) - } - } else if opBranch.Name == "x86_64" { - isExist := false - for _, fp := range opBranch.FullProductName { - if fp.FullProductName == pk.PackName && fp.Cpe == cpe { - isExist = true - break - } - } - if !isExist && strings.Contains(pk.PackName, ".x86_64.") { - var fullProductNamex86_64 FullProductName - fullProductNamex86_64.FullProductName = pk.PackName - fullProductNamex86_64.Cpe = cpe - if p, ok := checkRpm(pk.PackName); ok { - fullProductNamex86_64.ProductId = p - } - fullProductNameSlicex86_64 = append(fullProductNameSlicex86_64, fullProductNamex86_64) - } - } else { - isExist := false - for _, fp := range opBranch.FullProductName { - if fp.FullProductName == pk.PackName && fp.Cpe == cpe { - isExist = true - break - } - } - if !isExist && strings.Contains(pk.PackName, ".src.") { - var fullProductNamesrc FullProductName - fullProductNamesrc.FullProductName = pk.PackName - fullProductNamesrc.Cpe = cpe - if p, ok := checkRpm(pk.PackName); ok { - fullProductNamesrc.ProductId = p - } - fullProductNameSlicesrc = append(fullProductNameSlicesrc, fullProductNamesrc) - } - } - } - } - if opBranch.Name == "aarch64" { - opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSliceaarch64...) - } else if opBranch.Name == "noarch" { - opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSlicenoarch...) - } else if opBranch.Name == "x86_64" { - opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSlicex86_64...) - } else { - opBranch.FullProductName = append(opBranch.FullProductName, fullProductNameSlicesrc...) - } - } - openEulerBranchx = append(openEulerBranchx, opBranch) - } - cvrfsa.ProductTree.OpenEulerBranch = openEulerBranchx - } else { - var productTree ProductTree - productTree.Xmlns = "http://www.icasi.org/CVRF/schema/prod/1.1" - openEulerBranchSlice := make([]OpenEulerBranch, 0) - var openEulerBranch0 OpenEulerBranch - fullProductNameSlice0 := make([]FullProductName, 0) - var fullProductName0 FullProductName - fullProductName0.FullProductName = affectBranch - fullProductName0.ProductId = affectBranch - affectBranchListx := strings.Split(affectBranch, "-") - cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", - affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) - fullProductName0.Cpe = cpe - fullProductNameSlice0 = append(fullProductNameSlice0, fullProductName0) - openEulerBranch0.FullProductName = fullProductNameSlice0 - openEulerBranch0.Type = "Product Name" - openEulerBranch0.Name = affectBranchListx[0] - openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranch0) - fullProductNameSliceaarch64 := make([]FullProductName, 0) - fullProductNameSlicenoarch := make([]FullProductName, 0) - fullProductNameSlicesrc := make([]FullProductName, 0) - fullProductNameSlicex86_64 := make([]FullProductName, 0) - if len(pkg) > 0 { - for _, pk := range pkg { - if strings.Contains(pk.PackName, ".aarch64.") { - var fullProductNameaarch64 FullProductName - fullProductNameaarch64.FullProductName = pk.PackName - fullProductNameaarch64.Cpe = cpe - if p, ok := checkRpm(pk.PackName); ok { - fullProductNameaarch64.ProductId = p - } - fullProductNameSliceaarch64 = append(fullProductNameSliceaarch64, fullProductNameaarch64) - } else if strings.Contains(pk.PackName, ".src.") { - var fullProductNamesrc FullProductName - fullProductNamesrc.FullProductName = pk.PackName - fullProductNamesrc.Cpe = cpe - if p, ok := checkRpm(pk.PackName); ok { - fullProductNamesrc.ProductId = p - } - fullProductNameSlicesrc = append(fullProductNameSlicesrc, fullProductNamesrc) - } else if strings.Contains(pk.PackName, ".x86_64.") { - var fullProductNamex86_64 FullProductName - fullProductNamex86_64.FullProductName = pk.PackName - fullProductNamex86_64.Cpe = cpe - if p, ok := checkRpm(pk.PackName); ok { - fullProductNamex86_64.ProductId = p - } - fullProductNameSlicex86_64 = append(fullProductNameSlicex86_64, fullProductNamex86_64) - } else { - var fullProductNamenoarch FullProductName - fullProductNamenoarch.FullProductName = pk.PackName - fullProductNamenoarch.Cpe = cpe - if p, ok := checkRpm(pk.PackName); ok { - fullProductNamenoarch.ProductId = p - } - fullProductNameSlicenoarch = append(fullProductNameSlicenoarch, fullProductNamenoarch) - } - } - } - if len(fullProductNameSliceaarch64) > 0 { - var openEulerBranchaarch64 OpenEulerBranch - openEulerBranchaarch64.Type = "Package Arch" - openEulerBranchaarch64.Name = "aarch64" - openEulerBranchaarch64.FullProductName = fullProductNameSliceaarch64 - openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchaarch64) - } - if len(fullProductNameSlicenoarch) > 0 { - var openEulerBranchnoarch OpenEulerBranch - openEulerBranchnoarch.Type = "Package Arch" - openEulerBranchnoarch.Name = "noarch" - openEulerBranchnoarch.FullProductName = fullProductNameSlicenoarch - openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchnoarch) - } - if len(fullProductNameSlicesrc) > 0 { - var openEulerBranchsrc OpenEulerBranch - openEulerBranchsrc.Type = "Package Arch" - openEulerBranchsrc.Name = "src" - openEulerBranchsrc.FullProductName = fullProductNameSlicesrc - openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchsrc) - } - if len(fullProductNameSlicex86_64) > 0 { - var openEulerBranchx86_64 OpenEulerBranch - openEulerBranchx86_64.Type = "Package Arch" - openEulerBranchx86_64.Name = "x86_64" - openEulerBranchx86_64.FullProductName = fullProductNameSlicex86_64 - openEulerBranchSlice = append(openEulerBranchSlice, openEulerBranchx86_64) - } - productTree.OpenEulerBranch = openEulerBranchSlice - cvrfsa.ProductTree = &productTree - } -} - -func checkRpm(rpm string) (string, bool) { - // rmp: zopfli-debugsource-1.0.3-1.oe2203sp1.aarch64.rpm - t := strings.Split(rpm, ".") - if len(t) > 3 { - // return: zopfli-debugsource-1.0.3-1 - return strings.Join(t[:len(t)-3], "."), true - } - - return "", false -} - -func BuildVulnerability(vlLenth int, v models.ExcelExport, - componentMap map[string]ComponentInfo, cpe string, branchFlag int) []Vulnerability { - vulnerabilitySlice := make([]Vulnerability, 0) - var vulnerability Vulnerability - vulnerability.Xmlns = "http://www.icasi.org/CVRF/schema/vuln/1.1" - vulnerability.Ordinal = strconv.Itoa(vlLenth) - var cveNotes CveNotes - var cveNote CveNote - cveNote.Ordinal = strconv.Itoa(vlLenth) - cveNote.Type = "General" - cveNote.Title = "Vulnerability Description" - cveBrief := XmlSpecCharHand(v.CveBrief) - cveNote.Note = cveBrief - cveNote.XmlLnag = "en" - cveNotes.CveNote = &cveNote - vulnerability.CveNotes = &cveNotes - vulnerability.ReleaseDate = v.PublicDate - vulnerability.Cve = v.CveNum - var productStatuses ProductStatuses - var status Status - status.Type = "Fixed" - var productId ProductId - productId.ProductId = cpe - productIdSlice := make([]ProductId, 0) - productIdSlice = append(productIdSlice, productId) - status.ProductId = productIdSlice - productStatuses.Status = append(productStatuses.Status, &status) - vulnerability.ProductStatuses = &productStatuses - var threats Threats - var threat Threat - threat.Type = "Impact" - threat.Description = models.OpenEulerScoreProc(v.OpenEulerScore) - threats.Threat = &threat - vulnerability.Threats = &threats - var cVSSScoreSets CVSSScoreSets - var scoreSet ScoreSet - scoreSet.BaseScore = fmt.Sprintf("%.1f", v.OpenEulerScore) - scoreSet.Vector = v.OvectorVule - cVSSScoreSets.ScoreSet = &scoreSet - vulnerability.CvssScoreSets = &cVSSScoreSets - var remediations Remediations - var remediation Remediation - remediation.Type = "Vendor Fix" - remediation.Description = v.Summary - remediation.Date = v.PublicDate - ownedComponent := "" - if len(v.InfluenceComponent) > 1 { - ownedComponent = v.InfluenceComponent - } else { - ownedComponent = v.OwnedComponent - } - componentInfo := componentMap[ownedComponent] - localOpenEulerSANum := "" - if branchFlag == 1 { - localOpenEulerSANum = componentInfo.OpenEulerSANum - } else { - localOpenEulerSANum = componentInfo.CveNumMap[cpe].OpenEulerSANum - } - secLinkConfig := beego.AppConfig.String("reflink::openeuler_web") - remediation.Url = secLinkConfig + "/en/security/security-bulletins/detail/?id=" + localOpenEulerSANum - remediations.Remediation = &remediation - vulnerability.Remediations = &remediations - vulnerabilitySlice = append(vulnerabilitySlice, vulnerability) - return vulnerabilitySlice -} - -func BuildVulnerabilitySlice(vulnerability []Vulnerability, v models.ExcelExport, - affectBranch string, componentMap map[string]ComponentInfo, branchFlag int) []Vulnerability { - //affectBranchListx := strings.Split(affectBranch, "-") - cpe := affectBranch - if vulnerability != nil && len(vulnerability) > 0 { - cveExist := false - for _, vl := range vulnerability { - if vl.Cve == v.CveNum && strings.Contains(vl.Remediations.Remediation.Description, v.InfluenceComponent) { - cpeExist := false - - for si, status := range vl.ProductStatuses.Status { - for _, pid := range status.ProductId { - if pid.ProductId == cpe { - cpeExist = true - break - } - } - if !cpeExist { - var productId ProductId - productId.ProductId = cpe - if branchFlag == 1 { - vl.ProductStatuses.Status[si].ProductId = append(vl.ProductStatuses.Status[si].ProductId, productId) - } else { - productIdSlice := make([]ProductId, 0) - productIdSlice = append(productIdSlice, productId) - vl.ProductStatuses.Status[si].ProductId = productIdSlice - } - } - } - cveExist = true - break - } - } - if !cveExist { - vlLenth := len(vulnerability) + 1 - vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe, branchFlag) - if len(vulnerabilitySlice) > 0 { - vulnerability = append(vulnerability, vulnerabilitySlice...) - } - } - } else { - vlLenth := 1 - vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe, branchFlag) - vulnerability = append(vulnerability, vulnerabilitySlice...) - } - return vulnerability -} - -func BuildVulnerabilitySet(cvrfsa *CvrfSa, v models.ExcelExport, - affectBranch string, componentMap map[string]ComponentInfo, branchFlag int) { - vulnerability := make([]Vulnerability, 0) - if cvrfsa.Vulnerability != nil && len(cvrfsa.Vulnerability) > 0 { - for _, vuln := range cvrfsa.Vulnerability { - vulnerability = append(vulnerability, vuln) - } - } - vulnerabilityx := BuildVulnerabilitySlice(vulnerability, v, - affectBranch, componentMap, branchFlag) - cvrfsa.Vulnerability = vulnerabilityx -} - -func BuildCvrfXml(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, - cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, - pkg []models.Package, branchFlag int) { - curDate := common.GetCurDate() - branchList := BranchExist(affectBranch, cvrfFileList) - cvrfsa.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" - cvrfsa.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" - BuildDocumentTitle(cvrfsa, v, affectBranch, cvrfFileList, componentMap, curDate, branchList, branchFlag) - BuildDocumentRef(cvrfsa, v, componentMap, branchFlag, affectBranch) - BuildProductTree(cvrfsa, v, pkg, affectBranch, branchFlag) - BuildVulnerabilitySet(cvrfsa, v, affectBranch, componentMap, branchFlag) -} - -func QueryCveMd5(cveNum []string, OwnedComponent, openEulerSANum, branch string, fixFlag int8) (bool, string) { - sort.Strings(cveNum) - tmpOpenEulerSANum := "" - cveStr := strings.Join(cveNum, ",") - cveStrx := cveStr + "," + OwnedComponent - fileMd5 := common.EncryptMd5(cveStrx) - var cfr models.CvrfSaRecord - cfr.Md5 = fileMd5 - cfr.Branch = branch - tbErr := models.GetCvrfRecord(&cfr, "cur_md5", "branch") - if tbErr == nil { - tmpOpenEulerSANum = cfr.OpenEulerSANum - cfr.UpdateTime = common.GetCurTime() - cfr.Md5 = fileMd5 - cfr.OpenEulerSANum = openEulerSANum - cfr.Status = 1 - cfr.IsExport = 1 - cfr.CveNum = cveStr - cfr.PackName = OwnedComponent - cfr.AffectFlag = fixFlag - cfr.Branch = branch - updateErr := models.UpdateCvrfRecord(&cfr, "UpdateTime", - "OpenEulerSANum", "Status", "IsExport", "CveNum", "PackName", "AffectFlag", "Branch") - if updateErr != nil { - logs.Error("UpdateCvrfRecord, ", updateErr) - } - return true, tmpOpenEulerSANum - } else { - cfr.CreateTime = common.GetCurTime() - cfr.Md5 = fileMd5 - cfr.OpenEulerSANum = openEulerSANum - cfr.Status = 1 - cfr.IsExport = 1 - cfr.CveNum = cveStr - cfr.PackName = OwnedComponent - cfr.AffectFlag = fixFlag - cfr.Branch = branch - num, iErr := models.InsertCvrfRecord(&cfr) - if iErr != nil || num == 0 { - logs.Error("InsertCvrfRecord, ", iErr) - } - return false, tmpOpenEulerSANum - } -} - -func UpdateCvrfRecord(openeuler_sa_num string, isExport int8) { - var cfr models.CvrfSaRecord - cfr.UpdateTime = common.GetCurTime() - cfr.Status = 1 - cfr.IsExport = isExport - cfr.OpenEulerSANum = openeuler_sa_num - uErr := models.UpdateCvrfRecord(&cfr, "UpdateTime", "Status", "IsExport") - if uErr != nil { - logs.Error("UpdateCvrfRecord, ", uErr) - } -} - -func ProcCvrfFileName(fileName string) { - var cfr models.SaFileList - cfr.FileName = fileName - tbErr := models.GetCvrfFileName(&cfr, "file_name") - if tbErr == nil { - cfr.Status = 1 - cfr.UpdateTime = common.GetCurTime() - models.UpdateCvrfFileName(&cfr, "Status", "UpdateTime") - } else { - cfr.CreateTime = common.GetCurTime() - cfr.Status = 1 - cfr.FileName = fileName - num, iErr := models.InsertCvrfFileName(&cfr) - if iErr != nil || num == 0 { - logs.Error("InsertCvrfRecord, ", iErr) - } - } -} - -func RecordCrvfInfo(fileName, filex, newText string, fixFlag int8) error { - fileBytes, err := ioutil.ReadFile(fileName) - if err != nil { - fmt.Println("ioutil.ReadFile, error : ", err, fileName) - return err - } - fileStr := string(fileBytes) - fileMd5 := common.EncryptMd5(fileStr) - openEulerSANum := "" - if fixFlag == UNAFFECTFLAG { - openEulerSANum = fileMd5 - } else { - if len(filex) > 5 { - openEulerSANum = filex[5 : len(filex)-4] - } - } - fileContent := fileBytes - var buf bytes.Buffer - compressor, err := zlib.NewWriterLevelDict(&buf, zlib.BestCompression, fileBytes) - if err != nil { - fmt.Println("Compression failed") - } else { - compressor.Write(fileBytes) - compressor.Close() - fileContent = buf.Bytes() - } - fileConStr := baseStdEncode(fileContent) - var cfr models.CvrfSaRecord - cfr.OpenEulerSANum = openEulerSANum - if fixFlag == UNAFFECTFLAG { - cfr.Md5 = fileMd5 - } - //cfr.Md5 = fileMd5 - tbErr := models.GetCvrfRecord(&cfr, "openeuler_sa_num") - if tbErr != nil || cfr.Id == 0 { - cfr.CreateTime = common.GetCurTime() - if fixFlag == UNAFFECTFLAG { - cfr.Md5 = fileMd5 - cfr.CveNum = " " - cfr.PackName = " " - } - cfr.OpenEulerSANum = openEulerSANum - cfr.XmlContent = fileConStr - cfr.Status = 1 - cfr.IsExport = 1 - cfr.AffectFlag = fixFlag - cfr.SortOpenEulerSANum = newText - num, iErr := models.InsertCvrfRecord(&cfr) - if iErr != nil || num == 0 { - logs.Error("InsertCvrfRecord, ", iErr) - } - } else { - cfr.UpdateTime = common.GetCurTime() - if fixFlag == UNAFFECTFLAG { - cfr.Md5 = fileMd5 - } - cfr.XmlContent = fileConStr - cfr.Status = 1 - cfr.IsExport = 1 - cfr.AffectFlag = fixFlag - cfr.SortOpenEulerSANum = newText - uErr := models.UpdateCvrfRecord(&cfr, "UpdateTime", "UpdateMd5", "XmlContent", - "Status", "IsExport", "AffectFlag", "SortOpenEulerSANum") - if uErr != nil { - logs.Error("UpdateCvrfRecord, ", uErr) - } - } - return nil -} - // File Upload func PostFile(filename string, targetUrl string) error { upfileErr := ObsUploadFile(targetUrl, filename) @@ -1421,72 +443,6 @@ func PostFile(filename string, targetUrl string) error { return upfileErr } -// downlaod file -func GetFile(url, filePath string) error { - res, err := http.Get(url) - if err != nil { - logs.Error("GetFile1, err: ", err) - return err - } - f, err := os.Create(filePath) - if err != nil { - logs.Error("GetFile2, err: ", err) - return err - } - io.Copy(f, res.Body) - return nil -} - -func ReadWriteFile(filePath string, fileSlice []string) error { - if len(fileSlice) == 0 { - return errors.New("file content does not exist") - } - localDataSlice := make([]string, 0) - localDataSlice = common.RemoveDupString(fileSlice) - sort.Strings(localDataSlice) - fileContent := strings.Join(localDataSlice, "\n") - fisExist, _ := PathExists(filePath) - if fisExist { - os.Remove(filePath) - } - file, err := os.Create(filePath) - if err != nil { - logs.Error("open file err", err) - return err - } - defer file.Close() - _, err = file.Write([]byte(fileContent)) - if err != nil { - logs.Error(err) - } - return nil -} - -func ReadFileAll(filePath string) string { - file, err := os.Open(filePath) - if err != nil { - fmt.Println(err) - return "" - } - defer file.Close() - fileinfo, err := file.Stat() - if err != nil { - fmt.Println(err) - return "" - } - filesize := fileinfo.Size() - buffer := make([]byte, filesize) - - bytesread, err := file.Read(buffer) - if err != nil { - fmt.Println(err) - return "" - } - logs.Info("bytes read: ", bytesread) - logs.Info("bytestream to string: ", string(buffer)) - return string(buffer) -} - type ReplaceHelper struct { OldPath string NewPath string diff --git a/cve-vulner-manager/taskhandler/excel.go b/cve-vulner-manager/taskhandler/excel.go index 3552b32c68659ccc54c367be3a9d0d28158973f5..c16752519f6188838671a88deb4ff7eeeddcc104 100644 --- a/cve-vulner-manager/taskhandler/excel.go +++ b/cve-vulner-manager/taskhandler/excel.go @@ -1,21 +1,12 @@ package taskhandler import ( - "encoding/csv" - "encoding/json" "errors" "fmt" - "io" - "io/ioutil" - "net/http" "os" - "path" - "path/filepath" - "regexp" "strconv" "strings" "sync" - "time" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" @@ -26,12 +17,6 @@ import ( "cvevulner/util" ) -const CVRFFKEY = "cvrfFileKey" -const BRANCHSKEY = "opBranchsKey" -const UNAFFECTCVRFKEY = "unaffectcvrfkey" -const FIXEDFLAGE = 1 -const UNAFFECTFLAG = 2 - var releaseDate map[string]int64 // CveExcel Excel export client @@ -60,65 +45,6 @@ type IssueAndPkg struct { var fillLock sync.Mutex var wgTrigger sync.WaitGroup -// GenerateCveExcel Generate Excel documents based on data. -// param snPrefix means security notice prefix. -// param snSuffix means security notice suffix append start value. -// param forceRewrite means whether to force the document to be rewritten. -func GenerateCveExcel(excelName, snPrefix string, snSuffix int64, forceRewrite bool) (err error) { - //Query the data to be exported. - count := models.GetCanExportVulnCenterCount() - if count < 1 { - return errors.New("No data to export! ") - } - //init excelClient - ec := CveExcel{} - err = ec.Init(excelName, snPrefix, snSuffix) - if err != nil { - return err - } - mode := ec.InitFileHandle(forceRewrite) - if mode == 0 { - ec.InitSheet() - err = ec.FillHeader() - if err != nil { - logs.Error(err) - } - } - ec.FillContent(count) - //generate excel - return ec.Save(mode) -} - -// GenerateCveExcelByTrigger Generate cve&security notice excel file by trigger -func GenerateCveExcelByTrigger(affectBranch, excelName, snPrefix, startTime string, snSuffix int64, - forceRewrite bool, pkgList []models.ExcelPackage, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, cvexml *[]CveXml, - dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, - packRpmx map[string][]PackRpm, cves []string) (err error) { - if len(pkgList) == 0 { - return errors.New("No data to export! ") - } - //init excelClient - ec := CveExcel{} - err = ec.Init(excelName, snPrefix, snSuffix) - if err != nil { - logs.Error("excelName: ", excelName, err) - return err - } - mode := ec.InitFileHandle(forceRewrite) - if mode == 0 { - ec.InitSheet() - err = ec.FillHeader() - if err != nil { - logs.Error(err) - } - } - ec.FillContentTrigger(pkgList, startTime, affectBranch, cvrfFileList, - componentMap, cvfrFileMap, cvexml, dpdates, - securityNotice, packRpmx, cves) - return ec.Save(mode) -} - // Init init excel client func (ec *CveExcel) Init(excelName, snPrefix string, snSuffix int64) (err error) { if excelName == "" || !(strings.HasSuffix(excelName, ".xlsx") || strings.HasSuffix(excelName, "xls")) { @@ -382,65 +308,6 @@ func (ec *CveExcel) FillHeader() (err error) { return nil } -// FillContent fill the excel content -func (ec *CveExcel) FillContent(count int64) { - pageSize := 50 - pageCount := count / int64(pageSize) - if count%int64(pageSize) > 0 { - pageCount++ - } - for i := int64(0); i < pageCount; i++ { - off := i * int64(pageSize) - err := ec.handleWriteContent(off, pageSize) - if err != nil { - logs.Error(err) - } - } -} - -func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, - affectBranch string, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, cvexml *[]CveXml, - dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, - packRpmx map[string][]PackRpm, cves []string) { - pl := len(pkgList) - pageSize := 10 - pc := pl / 10 - if pl%10 > 0 { - pc++ - } - cd := make(chan []IssueAndPkg) - start := 0 - end := 0 - for i := 0; i < pc; i++ { - start = i * pageSize - end = (i + 1) * pageSize - if end > pl { - end = pl - } - wgTrigger.Add(1) - go getDateByGite(pkgList[start:end], startTime, cd, affectBranch, cves) - } - for i := 0; i < pc; i++ { - wgTrigger.Add(1) - go ec.handleGiteData(cd, affectBranch, cvexml, dpdates, - securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap) - } - wgTrigger.Wait() -} - -func FileExist(fileList []string) bool { - if len(fileList) > 0 { - for _, filex := range fileList { - _, isErr := os.Stat(filex) - if isErr != nil { - return false - } - } - } - return true -} - func DelFile(fileList []string) { if len(fileList) > 0 { for _, filex := range fileList { @@ -452,163 +319,6 @@ func DelFile(fileList []string) { } } -func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { - list, err := models.GetCanExportCveData(off, size) - if err != nil { - return err - } - var cvrfFileList map[string][]string - var componentMap map[string]ComponentInfo - var cvfrFileMap map[string]CvrfSa - lz := len(list) - if lz > 0 { - for _, v := range list { - if v.Num == 1 { - ec.setContentRow(v, "", cvrfFileList, componentMap, cvfrFileMap) - } else if v.Num > 1 { - //1.Obtain issue_tpl according to cve_num, if all issue_status == 2 0r issue_status == 6, then data can be exported - list, err := models.GetIssueTplByCveNum(v.CveNum) - if err != nil { - logs.Error(err) - continue - } - mergerList := make([]string, 0) - canMerger := true - for _, t := range list { - if t.Status != 3 && t.Status != 4 { - canMerger = false - break - } - if t.IssueStatus == 2 { - mergerList = append(mergerList, strconv.FormatInt(t.CveId, 10)) - } - } - //2.Find the corresponding ExcelExport data with the highest score from the template data of issue_status == 2 - if canMerger && len(mergerList) > 0 { - canExport, err := models.GetCanExportCveDataSameNum(strings.Join(mergerList, ",")) - if err != nil { - logs.Error(err) - } - ep := canExport[0] - if len(canExport) > 1 { - canExport = canExport[1:] - for _, ex := range canExport { - ep.Introduction = ep.Introduction + "\n" + ex.Introduction - ep.Summary = ep.Summary + "\n" + ex.Summary - ep.Theme = ep.Theme + "\n" + ex.Theme - ep.Description = ep.Description + "\n" + ex.Description - ep.OwnedComponent = ep.OwnedComponent + "\n" + ex.OwnedComponent - ep.InfluenceComponent = ep.InfluenceComponent + "\n" + ex.InfluenceComponent - ep.AffectProduct = ep.AffectProduct + "\n" + ex.AffectProduct - } - } - ec.setContentRow(ep, "", cvrfFileList, componentMap, cvfrFileMap) - } - } - } - } - return nil -} - -func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, - affectBranch string, cvexml *[]CveXml, dpdates *Updates, - securityNotice map[string][]SecurityNoticeXml, - packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) (err error) { - lz := len(list) - if lz > 0 { - for _, v := range list { - if v.OrganizateId == 2 { - logs.Error("opengauss, data: ", v) - continue - } - affectBool := affectBrachRep(&v, affectBranch) - if !affectBool { - logs.Error("Unaffected version, data: ", v) - continue - } - fillLock.Lock() - ec.setContentRow(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap) - fillLock.Unlock() - fillLock.Lock() - BuildXml(cvexml, &v, securityNotice, packRpmx, affectBranch) - fillLock.Unlock() - if v.Num == 1 { - // The current data num==1 will not be processed - //fillLock.Lock() - //BuildXml(cvexml, &v, securityNotice, packRpmx) - //fillLock.Unlock() - logs.Info("Num:1, info: ", v) - } else if v.Num > 1 { - logs.Info("Num: ", v.Num, ", info: ", v) - //Handling CVE_NUM duplicates - //1.Obtain issue_tpl according to cve_num, if all issue_status == 2 0r issue_status == 6, then data can be exported - list, err := models.GetIssueTplByCveNum(v.CveNum) - if err != nil { - logs.Error("GetIssueTplByCveNum, err: ", err) - continue - } - mergerList := make([]string, 0) - canMerger := true - for _, t := range list { - if t.Status != 3 && t.Status != 4 { - canMerger = false - break - } - if t.IssueStatus == 2 { - mergerList = append(mergerList, strconv.FormatInt(t.CveId, 10)) - } - } - if canMerger && len(mergerList) > 0 { - canExport, err := models.GetCanExportCveDataSameNum(strings.Join(mergerList, ",")) - if err != nil { - logs.Error("GetCanExportCveDataSameNum, err: ", err) - } - // Fill in the content of the xml file - canExport1 := make([]models.ExcelExport, len(canExport)) - copy(canExport1, canExport) - //addXmlData(canExport1, cvexml, affectBranch, securityNotice, packRpmx) - // Fill in the content of the excel file - ep := canExport[0] - ep.SecID = v.SecID - //affectBrachRep(&ep, affectBranch) - if ep.AffectProduct != "" && len(ep.AffectProduct) > 1 { - if ep.AffectProduct != affectBranch { - ep.Introduction = strings.ReplaceAll(ep.Introduction, ep.AffectProduct, affectBranch) - ep.Theme = strings.ReplaceAll(ep.Theme, ep.AffectProduct, affectBranch) - ep.AffectProduct = affectBranch - } - } - if len(canExport) > 1 { - canExport = canExport[1:] - m := make(map[string]struct{}) - m[ep.OwnedComponent] = struct{}{} - for _, ex := range canExport { - affectBool := affectBrachRep(&ex, affectBranch) - if !affectBool { - logs.Error("Unaffected version, data: ", ex) - continue - } - //component repeat do not append - if _, ok := m[ex.OwnedComponent]; !ok { - ep.Introduction = ep.Introduction + "\n" + ex.Introduction - ep.Summary = ep.Summary + "\n" + ex.Summary - ep.Theme = ep.Theme + "\n" + ex.Theme - ep.Description = ep.Description + "\n" + ex.Description - ep.InfluenceComponent = ep.InfluenceComponent + "\n" + ex.InfluenceComponent - ep.AffectProduct = ep.AffectProduct + "\n" + ex.AffectProduct - ep.OwnedComponent = ep.OwnedComponent + "\n" + ex.OwnedComponent - m[ex.OwnedComponent] = struct{}{} - } - } - } - } - } - } - } - return nil -} - func affectBrachRep(xmlp *models.ExcelExport, affectBranch string) bool { affectBool := false if xmlp.AffectProduct != "" && len(xmlp.AffectProduct) > 1 { @@ -670,338 +380,6 @@ func FindUnaffectBrach(xmlp *models.ExcelExport, affectBranch, accessToken, owne return affectBool } -func addXmlData(canExport []models.ExcelExport, cvexml *[]CveXml, affectBranch string, - securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) { - for _, xmlp := range canExport { - affectBool := affectBrachRep(&xmlp, affectBranch) - if !affectBool { - logs.Error("Unaffected version, data: ", xmlp) - continue - } - fillLock.Lock() - BuildXml(cvexml, &xmlp, securityNotice, packRpmx, affectBranch) - fillLock.Unlock() - } -} - -func UpdateUseOpenEulerSaNum() { - useOpenEulerNum := beego.AppConfig.String("xml::use_openeuler_num") - if len(useOpenEulerNum) > 1 { - useOpenEulerNumSlice := strings.Split(useOpenEulerNum, ",") - for _, use := range useOpenEulerNumSlice { - var sax models.SaNumber - sax.OpenEulerSANum = use - sax.Status = 1 - models.GetSaNumber(&sax, "OpenEulerSANum", "Status") - if sax.SaId > 0 { - sax.Status = 2 - sax.UpdateTime = common.GetCurTime() - models.UpdateSaNumber(&sax, "status") - } - var saFile models.SaFileList - saFile.FileName = "cvrf-" + use + ".xml" - models.GetCvrfFileName(&saFile, "FileName") - if saFile.FileId == 0 { - saFile.Status = 1 - saFile.CreateTime = common.GetCurTime() - models.InsertCvrfFileName(&saFile) - } - } - } -} - -func GetOpenEulerSaNum() string { - var sa models.SaNumber - curYears := strconv.Itoa(time.Now().Year()) - models.GetSaNumberByYears(&sa, curYears, 1, 1) - if sa.SaId > 0 { - sa.Status = 2 - sa.UpdateTime = common.GetCurTime() - models.UpdateSaNumber(&sa, "status") - } - return sa.OpenEulerSANum -} - -func StoreComponentInfo(componentMap map[string]ComponentInfo, - v models.ExcelExport, affectBranch string) bool { - repFlag := false - influenceComponent := "" - fileDir := beego.AppConfig.String("fileDir") - common.CreateDir(fileDir) - if len(v.InfluenceComponent) > 1 { - influenceComponent = v.InfluenceComponent - } else { - influenceComponent = v.OwnedComponent - } - packNameStruct, pOk := componentMap[influenceComponent] - if !pOk || len(packNameStruct.OpenEulerSANum) < 3 { - var coponentInfo ComponentInfo - openEulerSANum := GetOpenEulerSaNum() - if openEulerSANum == "" { - openEulerSANum = v.OpenEulerSANum - } - coponentInfo.OpenEulerSANum = openEulerSANum - coponentInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum+".xml") - coponentInfo.OwnedComponent = influenceComponent - coponentInfo.UpdateFlag = 2 - openEulerScoreSlice := make([]float64, 0) - openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) - coponentInfo.OpenEulerScore = openEulerScoreSlice - cveNumSlice := make([]string, 0) - cveNumSlice = append(cveNumSlice, v.CveNum) - branchCve := make(map[string]BrachCveInfo) - var brachCveInfo BrachCveInfo - openEulerSANum1 := GetOpenEulerSaNum() - if openEulerSANum1 == "" { - openEulerSANum1 = v.OpenEulerSANum - } - brachCveInfo.OpenEulerSANum = openEulerSANum1 - brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") - brachCveInfo.CveNumSlice = cveNumSlice - brachCveInfo.OpenEulerScore = openEulerScoreSlice - branchCve[affectBranch] = brachCveInfo - coponentInfo.CveNumMap = branchCve - coponentInfo.CveNum = cveNumSlice - componentMap[influenceComponent] = coponentInfo - } else { - packNameStruct.UpdateFlag = 2 - repFlagX := false - if len(packNameStruct.CveNum) > 0 { - for _, cveX := range packNameStruct.CveNum { - if cveX == v.CveNum { - repFlagX = true - break - } - } - if !repFlagX { - packNameStruct.CveNum = append(packNameStruct.CveNum, v.CveNum) - packNameStruct.OpenEulerScore = append(packNameStruct.OpenEulerScore, v.OpenEulerScore) - } - } else { - cveNumSlice := make([]string, 0) - cveNumSlice = append(cveNumSlice, v.CveNum) - packNameStruct.CveNum = cveNumSlice - openEulerScoreSlice := make([]float64, 0) - openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) - packNameStruct.OpenEulerScore = openEulerScoreSlice - } - if len(packNameStruct.CveNumMap) > 0 { - if branchCvex, brOk := packNameStruct.CveNumMap[affectBranch]; !brOk { - cveNumSlice := make([]string, 0) - cveNumSlice = append(cveNumSlice, v.CveNum) - openEulerScoreSlice := make([]float64, 0) - openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) - var brachCveInfo BrachCveInfo - openEulerSANum1 := GetOpenEulerSaNum() - if openEulerSANum1 == "" { - openEulerSANum1 = v.OpenEulerSANum - } - brachCveInfo.OpenEulerSANum = openEulerSANum1 - brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") - brachCveInfo.CveNumSlice = cveNumSlice - brachCveInfo.OpenEulerScore = openEulerScoreSlice - packNameStruct.CveNumMap[affectBranch] = brachCveInfo - } else { - for _, cve := range branchCvex.CveNumSlice { - if cve == v.CveNum { - repFlag = true - break - } - } - if !repFlag { - branchCvex.CveNumSlice = append(branchCvex.CveNumSlice, v.CveNum) - branchCvex.OpenEulerScore = append(branchCvex.OpenEulerScore, v.OpenEulerScore) - packNameStruct.CveNumMap[affectBranch] = branchCvex - } - } - } else { - branchCve := make(map[string]BrachCveInfo) - cveNumSlice := make([]string, 0) - cveNumSlice = append(cveNumSlice, v.CveNum) - openEulerScoreSlice := make([]float64, 0) - openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) - var brachCveInfo BrachCveInfo - openEulerSANum1 := GetOpenEulerSaNum() - if openEulerSANum1 == "" { - openEulerSANum1 = v.OpenEulerSANum - } - brachCveInfo.OpenEulerSANum = openEulerSANum1 - brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") - brachCveInfo.CveNumSlice = cveNumSlice - brachCveInfo.OpenEulerScore = openEulerScoreSlice - branchCve[affectBranch] = brachCveInfo - packNameStruct.CveNumMap = branchCve - } - componentMap[influenceComponent] = packNameStruct - } - return repFlag -} - -func SaveCvrfName(cvrfFileList map[string][]string, cvrffileName, mapKey string) { - cvrfFileSlice, cvrfOk := cvrfFileList[mapKey] - if cvrfOk { - fileExist := false - for _, cvrfVuale := range cvrfFileSlice { - if cvrfVuale == cvrffileName { - fileExist = true - } - } - if !fileExist { - cvrfFileSlice = append(cvrfFileSlice, cvrffileName) - cvrfFileList[mapKey] = cvrfFileSlice - } - } else { - cvrfNameSlice := make([]string, 0) - cvrfNameSlice = append(cvrfNameSlice, cvrffileName) - cvrfFileList[mapKey] = cvrfNameSlice - } -} - -func BranchCvrfData(v models.ExcelExport, - affectBranch, cvrffileName string, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, - cvfrFileMap map[string]CvrfSa, pkg []models.Package, branchFlag int) { - branchCvrfSaStruct, bCvrfSaOk := cvfrFileMap[cvrffileName] - if !bCvrfSaOk { - // Query whether there is data in the database - var cfr models.CvrfSaRecord - var cvrfsa CvrfSa - if branchFlag == 1 { - cfr.OpenEulerSANum = componentMap[v.InfluenceComponent].OpenEulerSANum - } else { - cfr.OpenEulerSANum = componentMap[v.InfluenceComponent].CveNumMap[affectBranch].OpenEulerSANum - } - tbErr := models.GetCvrfRecord(&cfr, "openeuler_sa_num") - if tbErr == nil && cfr.Id > 0 { - // Download data from file server - // Read file content - readErr := ReadCvrfXml(cvrffileName, &cvrfsa) - if readErr != nil { - cvrfsa = CvrfSa{} - } else { - var componentInfo ComponentInfo - componentInfo = componentMap[v.InfluenceComponent] - componentInfo.UpdateFlag = 1 - componentMap[v.InfluenceComponent] = componentInfo - } - } - BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) - cvfrFileMap[cvrffileName] = cvrfsa - } else { - BuildCvrfXml(&branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) - cvfrFileMap[cvrffileName] = branchCvrfSaStruct - } - SaveCvrfName(cvrfFileList, cvrffileName, CVRFFKEY) -} - -func procCvrfData(v models.ExcelExport, - affectBranch string, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, - cvfrFileMap map[string]CvrfSa, pkg []models.Package) { - totalCvrffileName := componentMap[v.InfluenceComponent].CvrfFileName - branchCvrffileName := componentMap[v.InfluenceComponent].CveNumMap[affectBranch].CvrfFileName - BranchCvrfData(v, affectBranch, branchCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 2) - BranchCvrfData(v, affectBranch, totalCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 1) -} - -func (ec *CveExcel) setContentRow(v models.ExcelExport, - affectBranch string, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { - pkg, pkgErr := models.GetCvePackageList(v.SecID, affectBranch) - if pkgErr != nil { - logs.Error(pkgErr) - } - if v.PublicDate == "" { - v.PublicDate = time.Now().Format("2006-01-02") - } - pkgStr := getPkgStr(pkg) - repFlag := StoreComponentInfo(componentMap, v, affectBranch) - if !repFlag { - procCvrfData(v, affectBranch, cvrfFileList, componentMap, cvfrFileMap, pkg) - sn := []interface{}{v.OpenEulerSANum, v.CveNum, v.Introduction, v.Summary, - v.Theme, v.Description, v.InfluenceComponent, - v.AffectProduct, pkgStr, v.ReferenceLink, v.PublicDate} - axis, searched := ec.searchValueInSheet(ec.SecNoticeSheetName, v.InfluenceComponent) - if !searched { - fillErr := ec.fillSecurityNoticeSheet(sn) - if fillErr != nil { - logs.Error(fillErr) - } - } else { - //merge openEuler SA notice data - colReg := regexp.MustCompile(`[A-Z]*`) - col := colReg.FindString(axis) - row := strings.Trim(axis, col) - rCN := "B" + row - rRl := "J" + row - vcn, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rCN) - vcn += ";\n" + v.CveNum - _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rCN, vcn) - vrl, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rRl) - vrl += "\n" + v.ReferenceLink - _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rRl, vrl) - rSAN := fmt.Sprintf("A%s", row) - vSAN, cellError := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rSAN) - if cellError == nil { - v.OpenEulerSANum = vSAN - } - //merger description - rd := "F" + row - vd, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rd) - dSplit := strings.Split(v.Description, "Security Fix(es):") - if len(dSplit) > 1 { - if !strings.Contains(vd, dSplit[0]) { - vd = dSplit[0] + vd - } - vd += dSplit[1] - } - _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rd, vd) - //Get the highest-rated theme - rd = "E" + row - theme, err := models.GetCanExportTheme(vcn, v.OwnedComponent, affectBranch) - if err == nil && len(theme) > 1 { - _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rd, theme) - } - } - cve := []interface{}{v.CveNum, v.CveBrief, v.NVDScore, v.OpenEulerScore, v.NattackVector, v.OattackVector, - v.NattackComplexity, v.OattackComplexity, v.NprivilegeRequired, v.OprivilegeRequired, v.NuserInteraction, - v.OuserInteraction, v.Nscope, v.Oscope, v.Nconfidentiality, v.Oconfidentiality, v.Nintegrity, v.Ointegrity, - v.Navailability, v.Oavailability, v.ScoreType, v.OpenEulerSANum, v.PublicDate} - shErr := ec.fillCveSheetRow(cve) - if shErr != nil { - logs.Error(shErr) - } - ap := []interface{}{v.CveNum, v.AffectProduct, v.InfluenceComponent, v.AffectStatus} - err := ec.fillAffectProductSheet(ap) - if err != nil { - logs.Error(err) - } - for _, v := range pkg { - pk := []interface{}{v.PackName, v.PackUrl} - if _, ok := ec.searchValueInSheet(ec.PackageURLSheetName, v.PackName); !ok { - err := ec.fillPackageSheet(pk) - if err != nil { - logs.Error(err) - } - } - } - } -} - -func getPkgStr(pkg []models.Package) string { - pkgStr := "" - if len(pkg) == 0 { - return pkgStr - } - var ps []string - for _, v := range pkg { - if v.PackName != "" { - ps = append(ps, v.PackName) - } - } - return strings.Join(ps, ";\n") -} - func (ec *CveExcel) searchValueInSheet(sheetName, value string) (axis string, searched bool) { if value == "" { return @@ -1077,33 +455,6 @@ func (ec *CveExcel) Save(md int8) error { } -// ExtractPackageData extract the package data by csv file -func ExtractPackageData(lp string) (pkgList []models.ExcelPackage, err error) { - pkgLock.Lock() - defer pkgLock.Unlock() - if lp == "" || path.Ext(lp) != ".CSV" { - return pkgList, errors.New("the file path is error") - } - file, err := os.Open(lp) - if err != nil { - return pkgList, err - } - defer file.Close() - reader := csv.NewReader(file) - for { - line, err := reader.Read() - if err == io.EOF { - break - } else if err != nil { - return pkgList, err - } - if len(line) >= 3 && len(line[0]) > 3 { - pkgList = append(pkgList, models.ExcelPackage{PubTime: line[0], Repo: line[1], Packages: line[2]}) - } - } - return -} - func getDataUnaffect(startTime string, cves []string) ([]models.IssueTemplate, error) { issueTemp, err := models.GetUnffectIssueNumber(startTime, cves) if err != nil || issueTemp == nil { @@ -1113,70 +464,6 @@ func getDataUnaffect(startTime string, cves []string) ([]models.IssueTemplate, e return issueTemp, err } -func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []IssueAndPkg, affectBranch string, cves []string) { - defer wgTrigger.Done() - token := beego.AppConfig.String("gitee::git_token") - //token := "8457c66db66955376519059b97e33dd1" - owner := beego.AppConfig.String("gitee::owner") - // Time difference in different time zones - saTimeStampZone, ok := beego.AppConfig.Int64("excel::sa_timestamp_zone") - if ok != nil { - saTimeStampZone = 3600 * 8 - } - st := util.TimeStrToInt(startTime, "2006-01-02") - if len(cves) != 0 { - st = 0 - } - chData := make([]IssueAndPkg, 0) - for _, v := range pkgList { - //logs.Info("excel, v===>", v) - rt := util.TimeStrToInt(v.PubTime, "20060102 15-04-05") + saTimeStampZone - // Query the current issues that need to be dealt with - issueTemp, err := models.GetIssueNumber(v.Repo, cves) - if err != nil || issueTemp == nil { - logs.Info("No need to deal with ", v.Repo) - continue - } - for _, isTemp := range issueTemp { - if len(cves) != 0 && !filterDataInSlice(isTemp.CveNum, cves) { - continue - } - //logs.Info("isTemp===>", isTemp) - prList := getRepoIssueAllPR(affectBranch, token, owner, v.Repo, st, rt, isTemp) - //get pull request related issue - repoIssue := make(map[int64]models.PullRequestIssue, 0) - for _, p := range prList { - //getPRRelatedAllIssue(token, owner, v.Repo, st, rt, p.Number, repoIssue) - repoIssue[p.Id] = p - } - if len(repoIssue) > 0 { - chData = append(chData, IssueAndPkg{IssueMap: repoIssue, IssuePkg: v.Packages, - Repo: v.Repo, IssueId: isTemp.IssueId}) - } - } - } - c <- chData -} - -func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, cvexml *[]CveXml, - dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, - packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { - defer wgTrigger.Done() - data := <-c - var pkgList []string - for _, v := range data { - //parse package string to list - pkgList = strings.Split(v.IssuePkg, " ") - if len(pkgList) == 0 { - logs.Error("Data is filtered, v.IssuePkg: ", v.IssuePkg) - continue - } - affectIssueProc(v, affectBranch, cvexml, dpdates, securityNotice, packRpmx, - cvrfFileList, componentMap, cvfrFileMap, pkgList, ec) - } -} - func UnaffectIssueProc(affectBranch string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, startTime, accessToken, owner string, unaffectcvrf *UnaffectCvrfSa, unaffectYear int, cves []string) { @@ -1298,127 +585,6 @@ func filterFixBranch(data *models.ExcelExport, cve, branch string) (has bool) { return } -func affectIssueProc(v IssueAndPkg, affectBranch string, cvexml *[]CveXml, - dpdates *Updates, securityNotice map[string][]SecurityNoticeXml, - packRpmx map[string][]PackRpm, cvrfFileList map[string][]string, - componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa, - pkgList []string, ec *CveExcel) { - for _, iv := range v.IssueMap { - tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo, IssueId: v.IssueId} - err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo", "issue_id") - if err != nil { - logs.Error("GetIssueTemplateByColName, ----", err, iv.Number, iv.Repo, v.IssueId) - continue - } - //save data to excel - el, err := models.GetCanExportExcelData(tpl.CveNum, tpl.IssueNum, tpl.OwnedComponent, tpl.IssueId) - if err != nil { - logs.Error("GetCanExportExcelData, err: ", err, "tpl.CveNum, tpl.IssueNum: ", - tpl.CveNum, tpl.IssueNum, tpl.OwnedComponent) - continue - } - // Check whether the cve data has been released sa - exist, detail := GetCveSecurityNotice(tpl.CveNum, tpl.Repo, true) - if exist && detail.Result.IsFixed() { - if IsProductFixed(tpl.CveNum, v.Repo, affectBranch) { - continue - } - } - - // Branches that have been fixed before the release no longer publish announcements - if releaseTimeOfVersion, ok := releaseDate[affectBranch]; ok { - if iv.MergeAt < releaseTimeOfVersion { - tpl.IsIgnore = 1 - models.UpdateIssueTemplate(&tpl, "is_ignore") - continue - } - } - - err = models.ReplacePackageByCveId(pkgList, tpl.CveId, affectBranch) - if err != nil { - logs.Error("ReplacePackageByCveId, err: ", err, "tpl.CveId: ", tpl.CveId) - continue - } - err = ec.handleWriteContentSync(el, affectBranch, cvexml, dpdates, - securityNotice, packRpmx, cvrfFileList, componentMap, cvfrFileMap) - if err != nil { - logs.Error("handleWriteContentSync, err: ", err) - } - } -} - -func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, - releaseTime int64, isTemp models.IssueTemplate) (prList []models.PullRequestIssue) { - url := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/issues/%v/pull_requests", owner, isTemp.IssueNum) - req, err := http.NewRequest(http.MethodGet, url, nil) - if err != nil { - logs.Error("NewRequest, url: ", url, ",err: ", err) - return - } - q := req.URL.Query() - q.Add("access_token", token) - q.Add("repo", repo) - req.URL.RawQuery = q.Encode() - resp, err := http.DefaultClient.Do(req) - if err != nil { - logs.Error("DefaultClient, url: ", url, ",err: ", err) - return - } - if resp.StatusCode == http.StatusOK { - issuePr := make([]map[string]interface{}, 0) - read, err := ioutil.ReadAll(resp.Body) - if err != nil { - logs.Error("ReadAll, url: ", url, ",err: ", err) - return - } - resp.Body.Close() - err = json.Unmarshal(read, &issuePr) - if err != nil { - logs.Error("Unmarshal, url: ", url, ",err: ", err) - return - } - for _, v := range issuePr { - if _, ok := v["id"]; !ok { - continue - } - pr := models.PullRequestIssue{} - if v["state"].(string) == "merged" && v["mergeable"].(bool) { - //mt := v["closed_at"].(string).(time.Time).Local().Unix() - closedAt := v["closed_at"].(string) - mt := util.TimeStrToInt(closedAt[:19], "2006-01-02T15:04:05") - mergedAt := v["merged_at"].(string) - ct := util.TimeStrToInt(mergedAt[:19], "2006-01-02T15:04:05") - //logs.Info("======>>mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct, ",repo:", repo, ", pr: ", v) - //ct := v["merged_at"].(string).(time.Time).Local().Unix() - var pt int64 - if mt > 0 && ct > 0 { - if mt > ct { - pt = ct - } else { - pt = mt - } - - if pt >= startTime && pt <= releaseTime { - if v["base"].(map[string]interface{})["label"].(string) == affectBranch { - pr.Id = int64(v["id"].(float64)) - pr.Number = isTemp.IssueNum - pr.CveNumber = isTemp.CveNum - pr.Repo = repo - pr.BrFlag = true - pr.Branch = affectBranch - pr.MergeAt = pt - prList = append(prList, pr) - } - } - } - } - } - } else { - resp.Body.Close() - } - return -} - // CheckOpenEulerScoreAndVector checks if the OpenEulerScore and OpenEulerVector fields of the issue are set. // If either field is not set, it sends a comment to the issue and returns false. func CheckOpenEulerScoreAndVector(issue *models.IssueTemplate, comment func(content string)) bool { diff --git a/cve-vulner-manager/taskhandler/gauss.go b/cve-vulner-manager/taskhandler/gauss.go index 13b27fa0feccd68093bfe11e505bf4eb7f9f592a..471d46ef7efecc339cb7f174e465efa3331f3d10 100644 --- a/cve-vulner-manager/taskhandler/gauss.go +++ b/cve-vulner-manager/taskhandler/gauss.go @@ -168,113 +168,7 @@ func CreateGaussVersion(ogv *models.OpenGaussVersion, flag, affectStatus, Releas } func CreateReleaseCve(ogc *models.OpenGaussCveList, xmlp *models.ExcelExport, flag int, gaussId int64) []string { - updateSlice := make([]string, 0) - cveNumSlice := make([]string, 0) - if len(xmlp.CveNum) > 1 { - cveNumSlice = strings.Split(xmlp.CveNum, "-") - } - years := time.Now().Year() - if len(cveNumSlice) > 2 { - years, _ = strconv.Atoi(cveNumSlice[1]) - } - ogc.GaussId = gaussId - ogc.CveNum = xmlp.CveNum - updateSlice = append(updateSlice, "CveNum") - ogc.Description = xmlp.Description - updateSlice = append(updateSlice, "Description") - ogc.PackName = xmlp.OwnedComponent - updateSlice = append(updateSlice, "PackName") - ogc.RepoName = xmlp.Repo - updateSlice = append(updateSlice, "RepoName") - ogc.NVDScore = xmlp.NVDScore - updateSlice = append(updateSlice, "NVDScore") - ogc.OpenEulerScore = xmlp.OpenEulerScore - updateSlice = append(updateSlice, "OpenEulerScore") - ogc.NvectorVule = xmlp.NvectorVule - updateSlice = append(updateSlice, "NvectorVule") - ogc.OvectorVule = xmlp.OvectorVule - updateSlice = append(updateSlice, "OvectorVule") - if xmlp.NattackVector != "" { - ogc.NattackVector = common.StrFirstToUpper(strings.ToLower(xmlp.NattackVector)) - } - updateSlice = append(updateSlice, "NattackVector") - if xmlp.NaccessVector != "" { - ogc.NaccessVector = common.StrFirstToUpper(strings.ToLower(xmlp.NaccessVector)) - } - updateSlice = append(updateSlice, "NaccessVector") - ogc.OattackVector = xmlp.OattackVector - updateSlice = append(updateSlice, "OattackVector") - ogc.OaccessVector = xmlp.OaccessVector - updateSlice = append(updateSlice, "OaccessVector") - if xmlp.NattackComplexity != "" { - ogc.NattackComplexity = common.StrFirstToUpper(strings.ToLower(xmlp.NattackComplexity)) - } - updateSlice = append(updateSlice, "NattackComplexity") - if xmlp.NaccessComplexity != "" { - ogc.NaccessComplexity = common.StrFirstToUpper(strings.ToLower(xmlp.NaccessComplexity)) - } - updateSlice = append(updateSlice, "NaccessComplexity") - ogc.OattackComplexity = xmlp.OattackComplexity - updateSlice = append(updateSlice, "OattackComplexity") - ogc.OaccessComplexity = xmlp.OaccessComplexity - updateSlice = append(updateSlice, "OaccessComplexity") - if xmlp.NprivilegeRequired != "" { - ogc.NprivilegeRequired = common.StrFirstToUpper(strings.ToLower(xmlp.NprivilegeRequired)) - } - updateSlice = append(updateSlice, "NprivilegeRequired") - ogc.OprivilegeRequired = xmlp.OprivilegeRequired - updateSlice = append(updateSlice, "OprivilegeRequired") - if xmlp.NuserInteraction != "" { - ogc.NuserInteraction = common.StrFirstToUpper(strings.ToLower(xmlp.NuserInteraction)) - } - updateSlice = append(updateSlice, "NuserInteraction") - ogc.OuserInteraction = xmlp.OuserInteraction - updateSlice = append(updateSlice, "OuserInteraction") - if xmlp.Nscope != "" { - ogc.Nscope = common.StrFirstToUpper(strings.ToLower(xmlp.Nscope)) - } - updateSlice = append(updateSlice, "Nscope") - ogc.Oscope = xmlp.Oscope - updateSlice = append(updateSlice, "Oscope") - if xmlp.Nconfidentiality != "" { - ogc.Nconfidentiality = common.StrFirstToUpper(strings.ToLower(xmlp.Nconfidentiality)) - } - updateSlice = append(updateSlice, "Nconfidentiality") - ogc.Oconfidentiality = xmlp.Oconfidentiality - updateSlice = append(updateSlice, "Oconfidentiality") - if xmlp.Nintegrity != "" { - ogc.Nintegrity = common.StrFirstToUpper(strings.ToLower(xmlp.Nintegrity)) - } - updateSlice = append(updateSlice, "Nintegrity") - ogc.Ointegrity = xmlp.Ointegrity - updateSlice = append(updateSlice, "Ointegrity") - if xmlp.Navailability != "" { - ogc.Navailability = common.StrFirstToUpper(strings.ToLower(xmlp.Navailability)) - } - updateSlice = append(updateSlice, "Navailability") - ogc.Oavailability = xmlp.Oavailability - updateSlice = append(updateSlice, "Oavailability") - if xmlp.Nauthentication != "" { - ogc.Nauthentication = common.StrFirstToUpper(strings.ToLower(xmlp.Nauthentication)) - } - updateSlice = append(updateSlice, "Nauthentication") - ogc.Oauthentication = xmlp.Oauthentication - updateSlice = append(updateSlice, "Oauthentication") - ogc.ScoreType = xmlp.ScoreType - updateSlice = append(updateSlice, "ScoreType") - ogc.FixLabel = xmlp.IssueLabel - updateSlice = append(updateSlice, "FixLabel") - ogc.Status = 1 - updateSlice = append(updateSlice, "Status") - ogc.Years = years - updateSlice = append(updateSlice, "Years") - if flag == 2 { - ogc.UpdateTime = common.GetCurTime() - updateSlice = append(updateSlice, "UpdateTime") - } else { - ogc.CreateTime = common.GetCurTime() - } - return updateSlice + return nil } func GaussFindUnaffVersion(xmlp *models.ExcelExport, versions string, gi GitInfo) bool { @@ -876,7 +770,7 @@ func downloadGaussPackageFile(localPath, url string) error { } } -//ExtractPackageData extract the package data by excel file +// ReadExcelFile reads an Excel file and extracts package data. func ReadExcelFile(lp string, versions string) (pkgList []models.GaussExcelTag, err error) { pkgLock.Lock() defer pkgLock.Unlock() diff --git a/cve-vulner-manager/taskhandler/xml.go b/cve-vulner-manager/taskhandler/xml.go index f15fe4c825dc74aade92efaaab787d27f90ffd97..56f0c85bc5157c1b0d740f889c06b04204dc4015 100644 --- a/cve-vulner-manager/taskhandler/xml.go +++ b/cve-vulner-manager/taskhandler/xml.go @@ -2,18 +2,6 @@ package taskhandler import ( "encoding/xml" - "fmt" - "io/ioutil" - "os" - "sort" - "strings" - "time" - - "github.com/astaxie/beego/logs" - "github.com/xuri/excelize/v2" - - "cvevulner/common" - "cvevulner/models" ) type Updates struct { @@ -97,342 +85,3 @@ type CveXml struct { SecurityNotice map[string][]SecurityNoticeXml PackRpmx map[string][]PackRpm } - -func ReadXml(filePath string, dpdates *Updates) error { - if filePath == "" || len(filePath) == 0 { - filePath = "download/updateinfo.xml" - } - fisExist, ferr := PathExists(filePath) - if !fisExist { - logs.Error(ferr) - return ferr - } - fd, err := os.Open(filePath) - if err != nil { - logs.Error("open file err : ", err, ",filePath: ", filePath) - return err - } - defer fd.Close() - fileContent, err := ioutil.ReadAll(fd) - if err != nil { - logs.Error("read file err : ", err, ", filePath: ", filePath) - return err - } - err = xml.Unmarshal(fileContent, dpdates) - if err != nil { - logs.Error("unmarshal err : ", err, ", fileContent: ", fileContent) - return err - } - return nil -} - -func AbiAffectedVersionBool(abiVersion, branch string) bool { - abiVersionList := []string{} - brandsGroup := strings.Split(abiVersion, ",") - if len(brandsGroup) > 0 { - for _, brand := range brandsGroup { - if brand == "" || len(brand) < 2 { - continue - } - brand = common.BranchVersionRep(brand) - brandList := strings.Split(brand, ":") - if len(brandList) > 1 { - prams := strings.Replace(brandList[1], " ", "", -1) - prams0 := strings.Replace(brandList[0], " ", "", -1) - if prams == "是" && prams0 == branch { - abiVersionList = append(abiVersionList, brandList[0]) - break - } - } else { - brandList = strings.Split(brand, "：") - if len(brandList) > 1 { - prams := strings.Replace(brandList[1], " ", "", -1) - prams0 := strings.Replace(brandList[0], " ", "", -1) - if prams == "是" && prams0 == branch { - abiVersionList = append(abiVersionList, brandList[0]) - break - } - } - } - } - } - if len(abiVersionList) > 0 { - return true - } - return false -} - -func CreateUpdateExcel(excelPath string) string { - // File storage directory - sheetName := "cve_list" - xlsx := excelize.NewFile() - index, _ := xlsx.NewSheet(sheetName) - sheetTileMap := make(map[string]string) - sheetTileMap["A1"] = "cve编号" - sheetTileMap["B1"] = "issue编号" - sheetTileMap["C1"] = "issue所属仓库" - sheetTileMap["D1"] = "score" - sheetTileMap["E1"] = "version" - sheetTileMap["F1"] = "abi是否变化" - for k, v := range sheetTileMap { - xlsx.SetCellValue(sheetName, k, v) - } - xlsx.SetActiveSheet(index) - err := xlsx.SaveAs(excelPath) - if err != nil { - logs.Error(err) - return "" - } - return excelPath -} - -func CreateUpdateData(its models.IssueTemplate, affectBranch string) []interface{} { - ownedVersion := "" - cveData := make([]interface{}, 0) - cveData = append(cveData, its.CveNum) - cveData = append(cveData, its.IssueNum) - cveData = append(cveData, its.OwnedComponent) - cveData = append(cveData, its.OpenEulerScore) - ownedVersion = its.OwnedVersion - ori := models.QueryEulerOriginBRepo(its.OwnedComponent, affectBranch) - if len(ori) > 0 { - for _, or := range ori { - if len(or.Branchs) > 0 { - brSlice := strings.Split(or.Branchs, ",") - if len(brSlice) > 0 { - for _, br := range brSlice { - if br == affectBranch { - if len(or.Version) > 0 && len(or.Version) < 32 { - ownedVersion = or.Version - goto toVersion - } - } - } - } - } - } - } -toVersion: - cveData = append(cveData, ownedVersion) - abiBool := AbiAffectedVersionBool(its.AbiVersion, affectBranch) - if abiBool { - cveData = append(cveData, "是") - } else { - cveData = append(cveData, "否") - } - return cveData -} - -func ReadWriteUpdateExcel(excelPath, ownedComponent, cveNum, affectBranch string) error { - sheetName := "cve_list" - file, openErr := excelize.OpenFile(excelPath) - if openErr != nil { - logs.Error("fail to open the file, excelPath: ", excelPath, ", openErr: ", openErr) - return openErr - } - vc := models.QueryCveByPackName(cveNum, ownedComponent, 1) - if len(vc) > 0 { - for _, v := range vc { - issueTmp, err := models.GetIssueTemplateByStatuss(v.CveId) - if err == nil { - cveData := CreateUpdateData(issueTmp, affectBranch) - if len(cveData) > 0 { - rows, sheetErr := file.GetRows(sheetName) - if sheetErr != nil { - logs.Error(sheetErr) - } - idx := len(rows) + 1 - axis := fmt.Sprintf("A%d", idx) - setErr := file.SetSheetRow(sheetName, axis, &cveData) - if setErr != nil { - logs.Error("setErr: ", setErr) - } - } - } - } - } - fileErr := file.SaveAs(excelPath) - if fileErr != nil { - logs.Error("Failed to save file, ", fileErr) - } - return fileErr -} - -func WriteXml(filePath, excelName, affectBranch string, cveXmlList []CveXml, dpdates *Updates, - securityNotice map[string][]SecurityNoticeXml, packRpmx map[string][]PackRpm) { - for _, cveXml := range cveXmlList { - upDatex := Update{From: "openeuler.org", Type: "security", Status: "stable", Id: cveXml.OpenEulerSANum, - Title: cveXml.Introduction, Severity: cveXml.CveLevel, Release: "openEuler"} - var pubDate Issued - pubDate.Date = cveXml.PublicDate - upDatex.Issued = &pubDate - openEulerSANum := "" - var rfs References - if sn, ok := securityNotice[cveXml.OwnedComponent]; ok { - if len(sn) > 0 { - descriptionsList := make([]string, 0) - openEulerScoreSlice := make([]float64, 0) - for _, sec := range sn { - var rf Reference - rf.Id = sec.CveNum - rf.Title = sec.CveNum - rf.Href = sec.ReferenceLink - rf.Type = "cve" - descriptionsList = append(descriptionsList, sec.Description) - openEulerScoreSlice = append(openEulerScoreSlice, sec.openEulerScore) - rfs.Reference = append(rfs.Reference, rf) - if len(openEulerSANum) < 2 { - cfc := models.GetCvrfSaRecordByCve(sec.CveNum, affectBranch) - if len(cfc) > 0 { - for _, cf := range cfc { - if len(cf.SortOpenEulerSANum) > 1 { - openEulerSANum = cf.SortOpenEulerSANum - break - } - } - } - } - // write excel - ReadWriteUpdateExcel(excelName, cveXml.OwnedComponent, sec.CveNum, affectBranch) - } - upDatex.Description = strings.Join(descriptionsList, " ") - if len(openEulerScoreSlice) > 1 { - sort.Float64s(openEulerScoreSlice) - } - cveLevel := models.OpenEulerScoreProc(openEulerScoreSlice[len(openEulerScoreSlice)-1]) - if strings.ToLower(cveLevel) == "low" { - upDatex.Severity = "Low" - } else if strings.ToLower(cveLevel) == "medium" { - upDatex.Severity = "Moderate" - } else if strings.ToLower(cveLevel) == "high" { - upDatex.Severity = "Important" - } else { - upDatex.Severity = "Critical" - } - } - } - if len(openEulerSANum) > 2 { - upDatex.Id = openEulerSANum - } - upDatex.References = &rfs - var pl Pkglist - var ct Collection - ct.Name = "openEuler" - if prx, ok := packRpmx[cveXml.OwnedComponent]; ok { - if len(prx) > 0 { - for _, pr := range prx { - if strings.Contains(pr.PackName, ".src.") { - continue - } - var pe Package - pe.Filename = pr.PackName - //pe.Name = cveXml.OwnedComponent - if len(pr.PackName) > len(cveXml.OwnedComponent) { - //packVersion := pr.PackName[len(cveXml.OwnedComponent) + 1: len(pr.PackName) - 4] - packVersionList := strings.Split(pr.PackName, "-") - if len(packVersionList) >= 3 { - pe.Version = packVersionList[len(packVersionList)-2] - rpmName := packVersionList[len(packVersionList)-1][:len(packVersionList[len(packVersionList)-1])-4] - lastIndex := strings.LastIndexAny(rpmName, ".") - if lastIndex != -1 { - pe.Release = rpmName[:lastIndex] - pe.Arch = rpmName[lastIndex+1:] - } - pe.Name = strings.Join(packVersionList[0:len(packVersionList)-2], "-") - } - } - ct.Package = append(ct.Package, pe) - } - } - } - pl.Collection = &ct - upDatex.Pkglist = &pl - dpdates.Updatex = append(dpdates.Updatex, upDatex) - } - xmlOutPut, outPutErr := xml.MarshalIndent(dpdates, "", " ") - if outPutErr == nil { - headerBytes := []byte(xml.Header) - xmlOutPutData := append(headerBytes, xmlOutPut...) - os.Remove(filePath) - ioutil.WriteFile(filePath, xmlOutPutData, os.ModePerm) - } else { - logs.Error(outPutErr) - } -} - -func BuildXml(cveXml *[]CveXml, v *models.ExcelExport, - securityNotice map[string][]SecurityNoticeXml, - packRpmx map[string][]PackRpm, affectBranch string) { - pkg, pkgErr := models.GetCvePackageList(v.SecID, affectBranch) - if pkgErr != nil { - logs.Error(pkgErr) - return - } - //cvex.SecurityNotice = make(map[string][]SecurityNoticeXml) - sn := SecurityNoticeXml{CveNum: v.CveNum, Title: v.CveNum, Type: "cve", ReferenceLink: v.ReferenceLink, - Description: deleteTailBlank(v.CveBrief) + "(" + v.CveNum + ")", openEulerScore: v.OpenEulerScore} - if vx, ok := securityNotice[v.OwnedComponent]; !ok { - snArry := []SecurityNoticeXml{} - snArry = append(snArry, sn) - securityNotice[v.OwnedComponent] = snArry - var cvex CveXml - if v.PublicDate == "" { - v.PublicDate = time.Now().Format("2006-01-02") - } - cvex.OpenEulerSANum = v.OpenEulerSANum - cvex.PublicDate = v.PublicDate - if strings.ToLower(v.CveLevel) == "low" { - cvex.CveLevel = "Low" - } else if strings.ToLower(v.CveLevel) == "medium" { - cvex.CveLevel = "Moderate" - } else if strings.ToLower(v.CveLevel) == "high" { - cvex.CveLevel = "Important" - } else { - cvex.CveLevel = "Critical" - } - if len(v.Introduction) > 1 { - cvex.Introduction = v.Introduction[:len(v.Introduction)-1] - } else { - cvex.Introduction = v.Introduction - } - cvex.OwnedComponent = v.OwnedComponent - *cveXml = append(*cveXml, cvex) - } else { - sameFlag := false - for _, vu := range vx { - if sn.CveNum == vu.CveNum { - sameFlag = true - break - } - } - if !sameFlag { - vx = append(vx, sn) - securityNotice[v.OwnedComponent] = vx - } - } - //cvex.PackRpmx = make(map[string][]PackRpm) - if px, ok := packRpmx[v.OwnedComponent]; !ok { - prArry := []PackRpm{} - for _, v := range pkg { - pr := PackRpm{PackName: v.PackName} - prArry = append(prArry, pr) - } - packRpmx[v.OwnedComponent] = prArry - } else { - for _, v := range pkg { - pnFlag := false - for _, p := range px { - if p.PackName == v.PackName { - pnFlag = true - break - } - } - if !pnFlag { - pr := PackRpm{PackName: v.PackName} - px = append(px, pr) - } - } - packRpmx[v.OwnedComponent] = px - } - return -}