diff --git a/conf/app.conf b/conf/app.conf index ffe86d19b3b6dd26d49b248aa65259d9c277d10c..ddf4f3252b2397712dba2aab105ef18bb2d906f3 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -116,4 +116,6 @@ comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual. [excel] forcerewrite = false snprefix = op-2020-10- -snsuffix = 1002 \ No newline at end of file +snsuffix = 1002 +# Version package excel download address +v_pack_excel_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv \ No newline at end of file diff --git a/conf/product_app.conf b/conf/product_app.conf index b8ca6edf8081f93db6b6748792a143039d537528..ee404c7c1d36e33395e4e7288452e689b811cd99 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -113,4 +113,6 @@ comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual. [excel] forcerewrite = false snprefix = op-2020-10- -snsuffix = 1002 \ No newline at end of file +snsuffix = 1002 +# Version package excel download address +v_pack_excel_url = http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv \ No newline at end of file diff --git a/controllers/file.go b/controllers/file.go index 1fdc2fdac2ba004808e2e52365883883e8e34f64..0f3f53b4522fecb3190313475c9ca89c2874aa0a 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -10,6 +10,7 @@ import ( "github.com/astaxie/beego/logs" "path/filepath" "regexp" + "strings" "time" ) @@ -103,19 +104,29 @@ func (f *FileController) TriggerCveData() { f.Ctx.WriteString(`Error: please enter the correct start time in a format like this "yyyy-MM-dd".`) return } - now := time.Now().Unix() - en := fmt.Sprintf("cve与安全公告%v.xlsx", now) - fileCode := common.EncryptMd5(en) - //call the generate excel task by new thread - er = models.ExportRecord{FileName: en, FileCode: fileCode, State: 0, CreateTime: timeUnix} - err = er.Insert() - if err != nil { - f.Ctx.WriteString(fmt.Sprintf("error:%v", err)) + affectBranchsxList := []string{} + affectedBranchs := beego.AppConfig.String("cve::affected_branchs") + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") } else { - //return the success notice - go taskhandler.GenerateExcelTrigger(en, startTime, fileCode) - f.Ctx.WriteString(fmt.Sprintf("Success：The name of the excel file generated this time is: %s. "+ - "It takes some time to generate the excel file. "+ - "You can try to call the download file interface and pass in the param fileCode=%s to be downloaded.", en, fileCode)) + affectBranchsxList = append(affectBranchsxList, "openEuler-20.03-LTS") + } + for _, affectBranch := range affectBranchsxList { + now := time.Now().Unix() + en := fmt.Sprintf("cve与安全公告%v_%v.xlsx", affectBranch, now) + fileCode := common.EncryptMd5(en) + //call the generate excel task by new thread + er = models.ExportRecord{FileName: en, FileCode: fileCode, State: 0, CreateTime: timeUnix} + err = er.Insert() + if err != nil { + f.Ctx.WriteString(fmt.Sprintf("error:%v", err)) + } else { + //return the success notice + go taskhandler.GenerateExcelTrigger(en, startTime, fileCode, affectBranch) + f.Ctx.WriteString(fmt.Sprintf("Success：The name of the excel file generated this time is: %s. "+ + "It takes some time to generate the excel file. "+ + "You can try to call the download file interface and pass in the param fileCode=%s to be downloaded.", en, fileCode)) + } } + } diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index f8ab638a4aa84cf4c3b779c6125779aa48b917fd..2ee60b57ec4195f4392c4da00a30ea9d77a9cf07 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -572,7 +572,8 @@ func AddAffectBrands(branchVersion string) string { return branchs } -func CreateSecNoticeData(sec *models.SecurityNotice, iss models.VulnCenter, path, branchVersion string, opScore float64) { +func CreateSecNoticeData(sec *models.SecurityNotice, iss models.VulnCenter, + path, branchVersion string, opScore float64) { branchs := AddAffectBrands(branchVersion) sec.CveId = iss.CveId sec.CveNum = iss.CveNum diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 0501eb4bfcf93d7b5159594f4fb94ed098bcf0a3..dd1e6323a341d2843e6c54a34724e844652cbbd8 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -1049,7 +1049,7 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c issueTemp.CveId = cveid err := models.GetIssueTemplateByColName(&issueTemp, "cve_id") if err != nil { - logs.Info("no issueTemp: ",err) + logs.Info("no issueTemp: ", err) } issueTemp.CveNum = cveData.CveNumber issueTemp.OwnedComponent = lop.Components @@ -1182,7 +1182,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c score.NVDScore = nVDScore } openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) - if openError == nil && openEulerScore > 0{ + if openError == nil && openEulerScore > 0 { score.OpenEulerScore = openEulerScore } score.NvectorVule = lop.CvsVector @@ -1443,7 +1443,7 @@ func paraAffectBrandBool(affectedVersion string) bool { brandList := strings.Split(brand, ":") if len(brandList) > 1 { prams := strings.Replace(brandList[1], " ", "", -1) - if prams == "受影响" || prams == "不受影响"{ + if prams == "受影响" || prams == "不受影响" { unaffectedBranchList = append(unaffectedBranchList, brandList[0]) } } else { @@ -1616,13 +1616,13 @@ func GetCveSecurityNotice(cveNumber string) bool { logs.Error(err) return false } - logs.Info("url: ", req.URL.String(), "获取官网数据： ", body) var detail models.RespCveDetail err = json.Unmarshal(body, &detail) if err != nil { logs.Error(err) return false } + logs.Info("url: ", req.URL.String(), "获取openEuler官网数据： ", detail) if detail.Result != nil && detail.Result.Id > 0 { return true } @@ -1637,6 +1637,7 @@ func FilterCveExported() { logs.Error(err) return } + logs.Info("data: ", data) for _, v := range data { go func(center models.VulnCenter) { ewg.Add(1) @@ -1675,48 +1676,53 @@ func GenerateExcelTask1() error { } //GenerateExcelTrigger generate cve&security notice excel file by pr merge and influence package release. -func GenerateExcelTrigger(fileName, startTime, fileCode string) { +func GenerateExcelTrigger(fileName, startTime, fileCode, affectBranch string) { //FilterCveExported() - logs.Error("start....") + logs.Info(affectBranch, ", GenerateExcelTrigger start....") dir := beego.AppConfig.DefaultString("fileDir", "download") err := util.MakeDir(dir) if err != nil { logs.Error(err) } - fr := models.ExportRecord{FileName: fileName} - err = fr.Read("file_name") - if err != nil { - logs.Error("sddds", err) - return - } - fileName = filepath.Join(dir, fileName) - du := "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" - du = beego.AppConfig.DefaultString("rpUrl", du) - localPath := filepath.Join(dir, "release-package.CSV") - err = downloadPackageFile(localPath, du) - if err != nil { - logs.Error(err) - fr.State = 2 - _ = fr.Update("state") - return - } - pkgList, err := ExtractPackageData(localPath) - if err != nil { - logs.Error(err) - fr.State = 2 - } else { - su := time.Now().Format("2006-01-02") - snPrefix := "openEuler-" + su - snSuffix := int64(1001) - err = GenerateCveExcelByTrigger(fileName, snPrefix, startTime, snSuffix, true, pkgList) - logs.Error("end....") + if affectBranch == "openEuler-20.03-LTS" { + fr := models.ExportRecord{FileName: fileName} + err = fr.Read("file_name") + if err != nil { + logs.Error("sddds", err) + return + } + fileName = filepath.Join(dir, fileName) + du := "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" + //du := beego.AppConfig.String("excel::v_pack_excel_url") + du = beego.AppConfig.DefaultString("rpUrl", du) + localPath := filepath.Join(dir, "release-package.CSV") + err = downloadPackageFile(localPath, du) if err != nil { logs.Error(err) fr.State = 2 + _ = fr.Update("state") + return } - fr.State = 1 + pkgList, err := ExtractPackageData(localPath) + if err != nil { + logs.Error(err) + fr.State = 2 + } else { + su := time.Now().Format("2006-01-02") + snPrefix := "openEuler-" + su + snSuffix := int64(1001) + err = GenerateCveExcelByTrigger(affectBranch, fileName, snPrefix, startTime, snSuffix, true, pkgList) + if err != nil { + logs.Error(err) + fr.State = 2 + } + fr.State = 1 + } + _ = fr.Update("state") + } else { + logs.Error("affectBranch: ", affectBranch, ", 不能导出sa") } - _ = fr.Update("state") + logs.Info(affectBranch, ", GenerateExcelTrigger: end") } func downloadPackageFile(localPath, url string) error { diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 468c958a3f1aedfca0e10ee885d874d270fa6a50..882077425fe00e61f519a63f57da868bb23a1d4a 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -77,7 +77,7 @@ func GenerateCveExcel(excelName, snPrefix string, snSuffix int64, forceRewrite b } //GenerateCveExcelByTrigger Generate cve&security notice excel file by trigger -func GenerateCveExcelByTrigger(excelName, snPrefix, startTime string, snSuffix int64, +func GenerateCveExcelByTrigger(affectBranch, excelName, snPrefix, startTime string, snSuffix int64, forceRewrite bool, pkgList []models.ExcelPackage) (err error) { if len(pkgList) == 0 { return errors.New("No data to export! ") @@ -96,7 +96,7 @@ func GenerateCveExcelByTrigger(excelName, snPrefix, startTime string, snSuffix i logs.Error(err) } } - ec.FillContentTrigger(pkgList, startTime) + ec.FillContentTrigger(pkgList, startTime, affectBranch) return ec.Save(mode) } @@ -379,7 +379,7 @@ func (ec *CveExcel) FillContent(count int64) { } } -func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime string) { +func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime, affectBranch string) { pl := len(pkgList) pageSize := 10 pc := pl / 10 @@ -396,14 +396,14 @@ func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime end = pl } wgTrigger.Add(1) - go getDateByGite(pkgList[start:end], startTime, cd) + go getDateByGite(pkgList[start:end], startTime, cd, affectBranch) } for i := 0; i < pc; i++ { wgTrigger.Add(1) - go ec.handleGiteData(cd) + go ec.handleGiteData(cd, affectBranch) } wgTrigger.Wait() - logs.Error("cve_gen_finish") + logs.Info("cve_gen_finish") } func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { @@ -462,10 +462,31 @@ func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { return nil } -func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport) (err error) { +func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, affectBranch string) (err error) { lz := len(list) if lz > 0 { for _, v := range list { + affectBool := false + if v.AffectProduct != "" && len(v.AffectProduct) > 1 { + if v.AffectProduct != affectBranch { + affectBranchsxList := strings.Split(v.AffectProduct, "/") + for _, ab := range affectBranchsxList { + if ab == affectBranch { + affectBool = true + v.AffectProduct = affectBranch + v.Introduction = strings.ReplaceAll(v.Introduction, v.AffectProduct, affectBranch) + v.Theme = strings.ReplaceAll(v.Theme, v.AffectProduct, affectBranch) + break + } + } + } else { + affectBool = true + } + } + if !affectBool { + logs.Error("Unaffected version, data: ", v) + continue + } if v.Num == 1 { fillLock.Lock() ec.setContentRow(v) @@ -498,11 +519,39 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport) (err error } ep := canExport[0] ep.SecID = v.SecID + if ep.AffectProduct != "" && len(ep.AffectProduct) > 1 { + if ep.AffectProduct != affectBranch { + ep.AffectProduct = affectBranch + ep.Introduction = strings.ReplaceAll(ep.Introduction, ep.AffectProduct, affectBranch) + ep.Theme = strings.ReplaceAll(ep.Theme, ep.AffectProduct, affectBranch) + } + } if len(canExport) > 1 { canExport = canExport[1:] m := make(map[string]struct{}) m[ep.OwnedComponent] = struct{}{} for _, ex := range canExport { + affectBool := false + if ex.AffectProduct != "" && len(ex.AffectProduct) > 1 { + if ex.AffectProduct != affectBranch { + affectBranchsxList := strings.Split(ex.AffectProduct, "/") + for _, ab := range affectBranchsxList { + if ab == affectBranch { + affectBool = true + ex.AffectProduct = affectBranch + ex.Introduction = strings.ReplaceAll(ex.Introduction, ex.AffectProduct, affectBranch) + ex.Theme = strings.ReplaceAll(ex.Theme, ex.AffectProduct, affectBranch) + break + } + } + } else { + affectBool = true + } + } + if !affectBool { + logs.Error("Unaffected version, data: ", v) + continue + } //component repeat do not append if _, ok := m[ex.OwnedComponent]; !ok { ep.Introduction = ep.Introduction + "\n" + ex.Introduction @@ -719,7 +768,7 @@ func ExtractPackageData(lp string) (pkgList []models.ExcelPackage, err error) { return } -func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []IssueAndPkg) { +func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []IssueAndPkg, affectBranch string) { defer wgTrigger.Done() token := beego.AppConfig.String("gitee::git_token") //token := "8457c66db66955376519059b97e33dd1" @@ -729,7 +778,7 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I chData := make([]IssueAndPkg, 0) for _, v := range pkgList { rt := util.TimeStrToInt(v.PubTime, "20060102 15-04-05") - prList := getRepoAllPR(token, owner, v.Repo, st, rt) + prList := getRepoAllPR(affectBranch, token, owner, v.Repo, st, rt) //get pull request related issue repoIssue := make(map[int64]models.PullRequestIssue, 0) for _, p := range prList { @@ -742,12 +791,13 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I c <- chData } -func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg) { +func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string) { defer wgTrigger.Done() data := <-c var pkgList []string for _, v := range data { + logs.Info("The SA currently being generated is: ", v) //parse package string to list pkgList = strings.Split(v.IssuePkg, " ") if len(pkgList) == 0 { @@ -757,7 +807,7 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg) { tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo} err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo") if err != nil { - logs.Error("----",err) + logs.Error("----", err) continue } err = models.ReplacePackageByCveId(pkgList, tpl.CveId) @@ -771,7 +821,7 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg) { logs.Error(err) return } - err = ec.handleWriteContentSync(el) + err = ec.handleWriteContentSync(el, affectBranch) if err != nil { logs.Error(err) } @@ -779,7 +829,7 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg) { } } -func getRepoAllPR(token, owner, repo string, startTime, releaseTime int64) (prList []models.PullRequest) { +func getRepoAllPR(affectBranch, token, owner, repo string, startTime, releaseTime int64) (prList []models.PullRequest) { pageSize := 20 pageCount := 1 url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/pulls", owner, repo) @@ -793,7 +843,7 @@ func getRepoAllPR(token, owner, repo string, startTime, releaseTime int64) (prLi q.Add("sort", "created") q.Add("state", "merged") q.Add("per_page", strconv.Itoa(pageSize)) - q.Add("base", "openEuler-20.03-LTS") //target branch is openEuler-20.03-LTS + q.Add("base", affectBranch) //target branch is openEuler-20.03-LTS for { q.Del("page") q.Add("page", strconv.Itoa(pageCount)) diff --git a/taskhandler/grabissue.go b/taskhandler/grabissue.go index 3910d2bb988bd2ada2723dba34d5cdd816583827..1497ae46a1aabdcb41c80d0ebab1584bc2031c61 100644 --- a/taskhandler/grabissue.go +++ b/taskhandler/grabissue.go @@ -5,6 +5,7 @@ import ( "cvevulner/util" "encoding/json" "fmt" + "github.com/astaxie/beego" "github.com/astaxie/beego/logs" "io/ioutil" "net/http" @@ -129,12 +130,25 @@ func getInfProduct(token string, owner string, repo string) (infPro string, err if err != nil { return "", err } + affectBranchsxList := []string{} + affectedBranchs := beego.AppConfig.String("cve::affected_branchs") + if affectedBranchs != "" && len(affectedBranchs) > 0 { + affectBranchsxList = strings.Split(affectedBranchs, ",") + } + tempBrandList := []string{} for _, v := range branchList { - if strings.HasSuffix(v.Name, "-20.03-LTS") { - infPro = v.Name - break + if affectBranchsxList != nil && len(affectBranchsxList) > 0 { + for _, affectBrand := range affectBranchsxList { + if strings.HasSuffix(v.Name, affectBrand) { + tempBrandList = append(tempBrandList, v.Name) + break + } + } } } + if tempBrandList != nil && len(tempBrandList) > 0 { + infPro = strings.Join(tempBrandList, "/") + } return }