From 380cb7962f6df037a0d38169d19304e565dada06 Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Mon, 18 Jan 2021 15:27:30 +0800 Subject: [PATCH 1/3] 1. OpenGauss requirements; 2. Cvrf file generation requirements; 3. Issue statistics mail requirements; 4. Fix bugs; --- common/common.go | 64 ++- conf/app.conf | 17 +- conf/product_app.conf | 19 +- controllers/file.go | 152 ++++++- controllers/hook.go | 508 ++++++++++++++++++----- controllers/upload.go | 7 + cve-py/common/files.py | 54 +++ cve-py/common/times.py | 24 ++ cve-py/controller/taskcontroller.py | 26 +- cve-py/controller/timertaskcontroller.py | 4 + cve-py/downloadtask/downloadfiletask.py | 53 +++ cve-py/gauss_yaml/gauss_yaml.yaml | 252 +++++++++++ cve-py/issue_statistics/to_email.xlsx | Bin 0 -> 10612 bytes cve-py/issue_statistics/~$to_email.xlsx | Bin 0 -> 165 bytes cve-py/main.py | 2 - cve-py/tabletask/gauss_yaml.py | 238 +++++++++++ cve-py/tabletask/issue_statistics.py | 123 ++++++ models/cve.go | 83 +++- models/excel.go | 33 +- models/issue.go | 27 +- models/modeldb.go | 34 +- models/uploadcve.go | 7 + task/hook.go | 5 +- task/inittask.go | 32 ++ task/issuestatistics.go | 29 ++ task/issuetask.go | 215 ++++++---- task/oricvecheck.go | 23 +- taskhandler/assist.go | 6 +- taskhandler/common.go | 111 ++++- taskhandler/createissue.go | 10 + taskhandler/cve.go | 70 +++- taskhandler/cvrf.go | 185 ++++++++- taskhandler/excel.go | 104 ++++- taskhandler/hook.go | 5 +- taskhandler/issuestatistics.go | 156 +++++++ taskhandler/oricvecheck.go | 56 ++- taskhandler/sendemail.go | 45 +- util/parsepayload.go | 2 + 38 files changed, 2463 insertions(+), 318 deletions(-) create mode 100644 cve-py/common/files.py create mode 100644 cve-py/common/times.py create mode 100644 cve-py/gauss_yaml/gauss_yaml.yaml create mode 100644 cve-py/issue_statistics/to_email.xlsx create mode 100644 cve-py/issue_statistics/~$to_email.xlsx create mode 100644 cve-py/tabletask/gauss_yaml.py create mode 100644 cve-py/tabletask/issue_statistics.py create mode 100644 task/issuestatistics.go create mode 100644 taskhandler/issuestatistics.go diff --git a/common/common.go b/common/common.go index c1218f7..18d52cb 100644 --- a/common/common.go +++ b/common/common.go @@ -9,6 +9,8 @@ import ( "time" ) +const DATE_FORMAT = "2006-01-02 15:04:05" + func DesString(dbpwd string) (strs string) { defer Catchs() if "" != dbpwd || len(dbpwd) > 0 { @@ -34,7 +36,7 @@ func AesString(dbpwd string) (strs string) { strs, err := EnPwdCode(pwd, key) if err != nil { logs.Error(err) - }else { + } else { logs.Info(strs) } return strs @@ -131,17 +133,20 @@ func VectorParams(Vector string, VectorMap map[string]string) map[string]string } func GetCurTime() string { - return time.Now().Format("2006-01-02 15:04:05") + return time.Now().Format(DATE_FORMAT) } func GetCurDate() string { return time.Now().Format("2006-01-02") } +func GetCurDateFormat() string { + return time.Now().Format("2006/01/02") +} func GetBeforeTime(days int) string { nowTime := time.Now() getTime := nowTime.AddDate(0, 0, days) - resTime := getTime.Format("2006-01-02 15:04:05") // The format of the obtained time + resTime := getTime.Format(DATE_FORMAT) // The format of the obtained time logs.Info("获取：", days, "前的时间：", resTime) return resTime } @@ -352,7 +357,7 @@ func PraseTimeInt(stringTime time.Time) int64 { timeStr = timeStr[:19] } loc, _ := time.LoadLocation("Local") - theTime, err := time.ParseInLocation("2006-01-02 15:04:05", timeStr, loc) + theTime, err := time.ParseInLocation(DATE_FORMAT, timeStr, loc) if err == nil { unixTime := theTime.Unix() return unixTime @@ -376,4 +381,53 @@ func IsValueInList(value string, list []string) bool { } } return false -} \ No newline at end of file +} + +func GetLocalCurTime() string { + localTime := time.Unix(time.Now().Unix(), 0) + local, _ := time.LoadLocation("Asia/Shanghai") + timeFormat := DATE_FORMAT + return localTime.In(local).Format(timeFormat) +} + +func CompareSlice(a, b []string) bool { + if len(a) != len(b) { + return false + } + + if (a == nil) != (b == nil) { + return false + } + bFlag := false + for _, av := range a { + for _, bv := range b { + if av == bv { + bFlag = true + break + } else { + bFlag = false + } + } + } + return bFlag +} + +func TimeConverStr(timeStr string) string { + timeLayout := DATE_FORMAT + times, _ := time.Parse(timeLayout, timeStr) + timeUnix := times.Unix() + timx := time.Unix(timeUnix, 0).Format(timeLayout) + return timx +} + +func GetSpecialDate(beforeDate int) string { + year, month, date := time.Now().Date() + thisMonth := time.Date(year, month, beforeDate, 0, 0, 0, 0, time.Local) + startDate := "" + if date > beforeDate { + startDate = thisMonth.AddDate(0, 0, 0).Format(DATE_FORMAT) + } else { + startDate = thisMonth.AddDate(0, -1, 0).Format(DATE_FORMAT) + } + return startDate +} diff --git a/conf/app.conf b/conf/app.conf index 5061046..2e28a74 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -48,20 +48,20 @@ maxsize=204800 ymalflag = 2 getymal = 0 0 1 * * 1 cveflag = 2 -getcve = 0 34 11 * * * +getcve = 0 3 18 * * * oricveflag = 2 oricvecheck = 0 16 19 * * * getissueflag = 2 getissue = 0 40 15 * * * issueflag = 2 -createissue = 0 23 21 * * * +createissue = 0 7 16 * * * test = 0/10 * * * * * gittokenflag = 2 issueoath = * * */20 * * * genexcelflag = 2 genexcel = 0 0/10 * * * * days = -30 -prcnum = 50 +prcnum = 5 printlogflag = 1 printlog = 0 0/20 * * * * unlockflag = 2 @@ -70,7 +70,10 @@ hookflag = 2 prochook = 0 34 17 * * * exceptflag = 2 exceptcve = 0 3 19 * * * - +gensaflag = 2 +gensa = 0 3 19 * * * +issuestatisticflag = 1 +issuestatistic = 0 19 10 * * * [gitee] #owner = cve-test @@ -122,6 +125,10 @@ affected_branchs = openEuler-20.03-LTS close_issue_privilege = 2 # abnormal cve status ,Use "," to separate multiple states abn_cve_status =3 +# SA number, The remaining amount +sa_re_amount = 4000 +# The number of days the issue counts the mailing list +cve_statistics_date = 20 [reflink] @@ -140,6 +147,7 @@ sa_timestamp_zone = 28800 [xml] updateinfo_path = download/updateinfo.xml +cvrf_upload_path = http://159.138.2.2:9090/ [email] email_name = "${EMAIL_NAME||***}" @@ -149,5 +157,6 @@ email_port = 587 [opengauss] gauss_owner = opengauss +#gauss_owner = cve-gauss # git token git_gauss_token = "${GITEE_GAUSS_TOKEN||xxx}" \ No newline at end of file diff --git a/conf/product_app.conf b/conf/product_app.conf index 7673d6b..6181b27 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -69,8 +69,12 @@ unlockflag = 1 unlock = 0 0 12 * * * hookflag = 1 prochook = 0 1 8 * * * -exceptflag = 1 +exceptflag = 2 exceptcve = 0 0 4 * * * +gensaflag = 1 +gensa = 0 1 0 * * * +issuestatisticflag = 1 +issuestatistic = 0 0 9 * * 1,3 [gitee] @@ -118,7 +122,10 @@ affected_branchs = openEuler-20.03-LTS close_issue_privilege = 1 # abnormal cve status ,Use "," to separate multiple states abn_cve_status =3 - +# SA number, The remaining amount +sa_re_amount = 4000 +# The number of days the issue counts the mailing list +cve_statistics_date = 20 [reflink] comment_cmd = https://gitee.com/openeuler/cve-manager/blob/master/doc/md/manual.md @@ -136,9 +143,15 @@ sa_timestamp_zone = 28800 [xml] updateinfo_path = download/updateinfo.xml +cvrf_upload_path = http://localhost:9090/ [email] email_name = "${EMAIL_NAME||***}" email_pwd = "${EMAIL_PWD||***}" email_host = smtp.gmail.com -email_port = 587 \ No newline at end of file +email_port = 587 + +[opengauss] +gauss_owner = opengauss +# git token +git_gauss_token = "${GITEE_GAUSS_TOKEN||xxx}" \ No newline at end of file diff --git a/controllers/file.go b/controllers/file.go index 9437b8f..3dfc0b8 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -117,6 +117,7 @@ func (f *FileController) TriggerCveData() { componentMap := make(map[string]taskhandler.ComponentInfo) cvrfFileList := make(map[string][]string) cvrfFileMap := make(map[string]taskhandler.CvrfSa) + affectBranchSlice := make([]string, 0) du := "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" //du := beego.AppConfig.String("excel::v_pack_20_03_url") du = beego.AppConfig.DefaultString("excel::v_pack_20_03_url", du) @@ -126,6 +127,7 @@ func (f *FileController) TriggerCveData() { openBranchx := strings.Split(csvP, "@") if len(openBranchx) == 2 { affectBranch := openBranchx[0] + affectBranchSlice = append(affectBranchSlice, affectBranch) csvDownPath := openBranchx[1] now := time.Now().Unix() en := fmt.Sprintf("cve与安全公告%v_%v.xlsx", openBranchx[0], now) @@ -151,17 +153,80 @@ func (f *FileController) TriggerCveData() { } wgCrvf.Wait() // Generate cvrf file - GenAndUploadCvrf(cvrfFileMap, cvrfFileList) + GenAndUploadCvrf(cvrfFileMap, cvrfFileList, componentMap, affectBranchSlice, dir) } // generate and upload cvrf.xml -func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, cvrfFileList map[string][]string) { - if len(cvrfFileMap) > 0 { - for cvrfKey, cvrfVule := range cvrfFileMap { - taskhandler.WriteCvrfXml(cvrfKey, &cvrfVule) +func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, + cvrfFileList map[string][]string, componentMap map[string]taskhandler.ComponentInfo, + affectBranchSlice []string, dir string) { + //if len(cvrfFileMap) > 0 { + // for cvrfKey, cvrfVule := range cvrfFileMap { + // taskhandler.WriteCvrfXml(cvrfKey, &cvrfVule) + // } + //} + writeCvrfSlice := make([]string, 0) + totalFileSlice := make([]string, 0) + if len(componentMap) > 0 { + for comKey, comValue := range componentMap { + if len(affectBranchSlice) > 1 { + isEque := true + branchInfo1 := comValue.CveNumMap[affectBranchSlice[0]].CveNumSlice + for _, branchInfo := range affectBranchSlice[1:] { + if !common.CompareSlice(branchInfo1, comValue.CveNumMap[branchInfo].CveNumSlice) { + isEque = false + break + } + } + if isEque { + cvrfFileName := componentMap[comKey].CvrfFileName + cvrfVule := cvrfFileMap[cvrfFileName] + if !taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, + componentMap[comKey].OpenEulerSANum, taskhandler.FIXEDFLAGE) { + taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) + writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) + } else { + UpdateOpenEulerSaNumStatus(comValue.OpenEulerSANum) + } + for _, br := range comValue.CveNumMap { + UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) + } + } else { + for _, br := range comValue.CveNumMap { + cvrfFileName := br.CvrfFileName + cvrfVule := cvrfFileMap[br.CvrfFileName] + if !taskhandler.QueryCveMd5(br.CveNumSlice, componentMap[comKey].OwnedComponent, + br.OpenEulerSANum, taskhandler.FIXEDFLAGE) { + taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) + writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) + } else { + UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) + } + } + UpdateOpenEulerSaNumStatus(comValue.OpenEulerSANum) + } + } else { + cvrfFileName := componentMap[comKey].CvrfFileName + cvrfVule := cvrfFileMap[cvrfFileName] + if !taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, + componentMap[comKey].OpenEulerSANum, taskhandler.FIXEDFLAGE) { + taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) + writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) + } else { + UpdateOpenEulerSaNumStatus(comValue.OpenEulerSANum) + } + for _, br := range comValue.CveNumMap { + UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) + } + } } } - totalFileSlice := make([]string, 0) + cvrfFileList[taskhandler.CVRFFKEY] = writeCvrfSlice + uploadCvrfFile(cvrfFileList, totalFileSlice, dir) +} + +func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, dir string) { + uploadPath := beego.AppConfig.String("xml::cvrf_upload_path") // File name and data stored in database fileSlice, fOk := cvrfFileList[taskhandler.CVRFFKEY] if fOk { @@ -172,12 +237,41 @@ func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, cvrfFileList ma recordErr := taskhandler.RecordCrvfInfo(fPath, fileName, taskhandler.FIXEDFLAGE) logs.Info("recordErr: ", recordErr) // Upload file, pending - - subFileSlice = append(subFileSlice, fileName) - totalFileSlice = append(totalFileSlice, fPath) - // Upload successfully, modify file status - + uploadErr := taskhandler.PostFile(fPath, uploadPath) + if uploadErr == nil { + taskhandler.ProcCvrfFileName(fileName) + subFileSlice = append(subFileSlice, fileName) + totalFileSlice = append(totalFileSlice, fPath) + // Upload successfully, modify file status + if len(fileName) > 5 { + openEulerSANum := fileName[5 : len(fileName)-4] + taskhandler.UpdateCvrfRecord(openEulerSANum, 2) + } + } else { + logs.Error(uploadErr) + } } + totalSlice := make([]string, 0) + sfl := models.GetCvrfAllFile() + indexFilePath := filepath.Join(dir, "index.txt") + updateFilePath := filepath.Join(dir, "update_fixed.txt") + if len(sfl) > 0 { + for _, l := range sfl { + totalSlice = append(totalSlice, l.FileName) + } + //totalSlice = append(totalSlice, subFileSlice...) + readErr := taskhandler.ReadWriteFile(indexFilePath, totalSlice) + logs.Info(readErr) + } else { + readErr :=taskhandler.ReadWriteFile(indexFilePath, subFileSlice) + logs.Info(readErr) + } + taskhandler.PostFile(indexFilePath, uploadPath) + readErr :=taskhandler.ReadWriteFile(updateFilePath, subFileSlice) + logs.Info(readErr) + taskhandler.PostFile(updateFilePath, uploadPath) + totalFileSlice = append(totalFileSlice, indexFilePath) + totalFileSlice = append(totalFileSlice, updateFilePath) //if len(subFileSlice) > 0 { // taskhandler.SendCvrfEmail(subFileSlice) //} @@ -187,18 +281,29 @@ func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, cvrfFileList ma unaffFileSlice, unffOk := cvrfFileList[taskhandler.UNAFFECTCVRFKEY] if unffOk { subFileSlice := make([]string, 0) + updateFilePath := filepath.Join(dir, "update_unaffect.txt") for _, fPath := range unaffFileSlice { _, fileName := filepath.Split(fPath) // File storage to db recordErr := taskhandler.RecordCrvfInfo(fPath, fileName, taskhandler.UNAFFECTFLAG) logs.Info("recordErr: ", recordErr) // Upload file, pending - - subFileSlice = append(subFileSlice, fileName) - totalFileSlice = append(totalFileSlice, fPath) - // Upload successfully, modify file status - + uploadErr := taskhandler.PostFile(fPath, uploadPath) + if uploadErr == nil { + subFileSlice = append(subFileSlice, fileName) + totalFileSlice = append(totalFileSlice, fPath) + // Upload successfully, modify file status + if len(fileName) > 5 { + openEulerSANum := fileName[5 : len(fileName)-4] + taskhandler.UpdateCvrfRecord(openEulerSANum, 2) + } + } else { + logs.Error(uploadErr) + } } + taskhandler.ReadWriteFile(updateFilePath, subFileSlice) + taskhandler.PostFile(updateFilePath, uploadPath) + totalFileSlice = append(totalFileSlice, updateFilePath) } if len(totalFileSlice) > 0 { //taskhandler.SendCvrfEmail(totalFileSlice) @@ -210,7 +315,7 @@ func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, cvrfFileList ma logs.Error("File compression failed: err: ", zipErr) } // send email - sendError := taskhandler.SendEmail(zipFileName, 1) + sendError := taskhandler.SendEmail(zipFileName, 1, "") if sendError != nil { logs.Error("SendEmail, sendErr: ", sendError) return @@ -221,3 +326,16 @@ func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, cvrfFileList ma taskhandler.DelFile(totalFileSlice) logs.Info(cvrfFileList[taskhandler.BRANCHSKEY], ",End of generating cvrf format file this time") } + +func UpdateOpenEulerSaNumStatus(openEulerSANum string) { + var sa models.SaNumber + curYears := strconv.Itoa(time.Now().Year()) + sa.SaYears = curYears + sa.OpenEulerSANum = openEulerSANum + sa.UpdateTime = common.GetCurTime() + getErr := models.GetSaNumber(&sa, "OpenEulerSANum", "SaYears") + if getErr == nil { + sa.Status = 1 + models.UpdateSaNumber(&sa, "status") + } +} diff --git a/controllers/hook.go b/controllers/hook.go index 60b125c..c426b5e 100644 --- a/controllers/hook.go +++ b/controllers/hook.go @@ -280,6 +280,39 @@ func getMaintainer(path, prSender, assignee string) string { return maintainerVaule } +func gaussCloseIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, + token, owner, fixed, unFix, path string, cveCenter *models.VulnCenter) { + if _, tb, ok := checkGaussIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + issueTmp.IssueStatus = 1 + issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, unFix) + issueTmp.StatusName = "open" + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, path, + *cveCenter, *issueTmp) + if issueErr == nil { + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(ContentReview, issueTmp.Assignee) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + } + } else { + //1. change issue status + issueTmp.IssueStatus = 2 + //issueTmp.Status = 3 + cveCenter.IsExport = 3 + issueTmp.StatusName = issueHook.Issue.StateName + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(unFix, fixed) + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, unFix) + } + } +} + func closeIssueProc(issueHook *models.IssuePayload, issueTmp *models.IssueTemplate, token, owner, fixed, unFix string, cveCenter *models.VulnCenter) { closePrBool := true @@ -370,7 +403,9 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { issueTmp := models.IssueTemplate{} issueTmp.IssueId = issueId issueTmp.IssueNum = issueHook.Iid - if issueHook.Issue.Repository.Path != "" && len(issueHook.Issue.Repository.Path) > 1 { + if issueHook.Issue.Repository.Path != "" && + len(issueHook.Issue.Repository.Path) > 1 && + issueHook.Issue.Repository.Path != "security" { issueTmp.OwnedComponent = issueHook.Issue.Repository.Path issueErr := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "owned_component", "issue_id") if issueErr != nil { @@ -387,8 +422,16 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { if cveErr != nil { return cveErr } + path := issueTmp.Repo token := beego.AppConfig.String("gitee::git_token") owner := beego.AppConfig.String("gitee::owner") + gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") + gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") + if cveCenter.OrganizationID == 2 { + owner = gaussOwner + token = gitGaussToken + path = "security" + } issueTmp.StatusName = issueHook.Issue.StateName logs.Info("发起issue状态修改, 发起人: @", issueHook.Sender.UserName, ", 修改状态: ", issueHook.Issue.StateName, ", data: ", issueHook) @@ -396,7 +439,11 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { case IssueOpenState: issueTmp.Status = 1 cveCenter.IsExport = 0 - _, _, ok := checkIssueAnalysisComplete(&issueTmp) + checkFunc := checkIssueAnalysisComplete + if cveCenter.OrganizationID == 2 { + checkFunc = checkGaussIssueAnalysisComplete + } + _, _, ok := checkFunc(&issueTmp) if ok { issueTmp.IssueStatus = 3 } else { @@ -407,7 +454,11 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { case IssueProgressState: issueTmp.Status = 2 cveCenter.IsExport = 0 - _, _, ok := checkIssueAnalysisComplete(&issueTmp) + checkFunc := checkIssueAnalysisComplete + if cveCenter.OrganizationID == 2 { + checkFunc = checkGaussIssueAnalysisComplete + } + _, _, ok := checkFunc(&issueTmp) if ok { issueTmp.IssueStatus = 3 } else { @@ -420,13 +471,20 @@ func handleIssueStateChange(issueHook *models.IssuePayload) error { logs.Error("The issue has been closed and cannot be operated again,issuetmp: ", issueTmp) return errors.New("The issue has been closed and cannot be operated again") } - closeIssueProc(issueHook, &issueTmp, token, owner, fixed, unFix, &cveCenter) + if cveCenter.OrganizationID == 2 { + gaussCloseIssueProc(issueHook, &issueTmp, token, owner, fixed, unFix, path, &cveCenter) + } else { + closeIssueProc(issueHook, &issueTmp, token, owner, fixed, unFix, &cveCenter) + } case IssueRejectState: issueTmp.Status = 4 issueTmp.IssueStatus = 6 cveCenter.IsExport = 2 issueTmp.IssueLabel = issueHook.Issue.ReplaceLabelToStr(fixed, uNaffected) } + if cveCenter.OrganizationID == 2 { + issueTmp.Repo = path + } updateBool := updateTempAndCenter(issueTmp, cveCenter, token, owner) if !updateBool { return errors.New("handle issue state hook appear error maybe some step fail") @@ -767,7 +825,10 @@ func isNormalCloseIssue(cveID int64, issueState int8) bool { // update data func updateTempAndCenter(issueTmp models.IssueTemplate, cveCenter models.VulnCenter, token, owner string) bool { appearErr := 0 + isAffectFlag := true uNaffected := beego.AppConfig.String("labeUnaffected") + labelFixed := beego.AppConfig.String("labelFixed") + labelUnFix := beego.AppConfig.String("labelUnFix") sn := models.SecurityNotice{CveId: issueTmp.CveId, CveNum: issueTmp.CveNum} snErr := sn.Read("cve_id", "cve_num") if snErr != nil { @@ -782,6 +843,9 @@ func updateTempAndCenter(issueTmp models.IssueTemplate, cveCenter models.VulnCen } if sn.AffectProduct != "" && len(sn.AffectProduct) > 1 { affectProductList := strings.Split(sn.AffectProduct, "/") + if len(affectProductList) > 0 { + isAffectFlag = false + } for _, brands := range affectProductList { if len(affectBranchsxList) > 0 { for _, affectBranch := range affectBranchsxList { @@ -824,8 +888,14 @@ func updateTempAndCenter(issueTmp models.IssueTemplate, cveCenter models.VulnCen logs.Error("update vulnCenter fail ") appearErr += 1 } - if strings.Contains(issueTmp.IssueLabel, uNaffected) { + if strings.Contains(issueTmp.IssueLabel, uNaffected) && isAffectFlag { issueTmp.IssueLabel = uNaffected + } else { + if strings.Contains(issueTmp.IssueLabel, labelFixed) && issueTmp.IssueStatus == 2 { + issueTmp.IssueLabel = labelFixed + } else { + issueTmp.IssueLabel = labelUnFix + } } update = ChangeIssueLabel(token, issueTmp.Repo, issueTmp.IssueNum, owner, issueTmp.IssueLabel) if !update { @@ -863,6 +933,44 @@ func openEulerScoreReview(issueTmp *models.IssueTemplate, cuAccount, owner, toke return approveFlag } +func gaussMaintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, + path string, cveCenter models.VulnCenter) { + if _, tb, ok := checkGaussIssueClosedAnalysisComplete(issueTmp); !ok { + //send comment to issue + na := "\n**请确认分析内容的准确性,待分析内容请填写完整,否则将无法关闭当前issue.**" + cc := fmt.Sprintf(AnalysisComplete, cuAccount) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, token) + return + } else { + issueTmp.IssueLabel = fixed + issueTmp.StatusName = "closed" + taskhandler.AddCommentToIssue(fmt.Sprintf(`@%v 你已审核模板内容, cve-manager 将关闭issue!`, + cuAccount), issueTmp.IssueNum, owner, path, token) + _, issueErr := taskhandler.UpdateIssueToGit(token, owner, issueTmp.Repo, + cveCenter, *issueTmp) + if issueErr == nil { + logs.Info("发起issue关闭,issuetmp: ", issueTmp) + } else { + logs.Error("issue关闭操作失败, issuetmp: ", issueTmp, ",issueErr: ", issueErr) + return + } + //issueTmp.SaAuditFlag = 1 + issueTmp.Status = 3 + if isNormalCloseIssue(issueTmp.CveId, issueTmp.IssueStatus) { + issueTmp.IssueStatus = 2 + cveCenter.IsExport = 3 + } else { + issueTmp.IssueStatus = 6 + cveCenter.IsExport = 2 + } + issueTmp.Repo = "security" + updateBool := updateTempAndCenter(*issueTmp, cveCenter, token, owner) + if !updateBool { + return + } + } +} + func maintainerApprove(issueTmp *models.IssueTemplate, cuAccount, owner, token, fixed, unfixed string) { if _, tb, ok := checkIssueClosedAnalysisComplete(issueTmp); !ok { //send comment to issue @@ -990,17 +1098,33 @@ func handleIssueComment(payload models.CommentPayload) { logs.Error("Data has null values: issueNum, cuAccount, cBody: ", issueNum, cuAccount, cBody) return } + issueTmp := models.IssueTemplate{IssueNum: issueNum, IssueId: issueId} + err := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") + if err != nil { + logs.Error(err) + return + } + accessToken := os.Getenv("GITEE_TOKEN") + owner := beego.AppConfig.String("gitee::owner") + fixed := beego.AppConfig.String("labelFixed") + unfixed := beego.AppConfig.String("labelUnFix") + path := issueTmp.Repo + gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") + gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") + var vc models.VulnCenter + vcErr := models.GetVulnCenterByCVEID(&vc, issueTmp.CveId, "organizate_id", "cve_id") + if vcErr == nil && vc.OrganizationID == 2 { + owner = gaussOwner + accessToken = gitGaussToken + path = "security" + cBody = strings.ReplaceAll(cBody, util.KwOpenGaussScore, util.KwOpenEulerScore) + } + if strings.HasPrefix(cBody, cmdRej) { //Review rejected Add comment @Analyst if !isReviewer(cuAccount) { return } - issueTmp := models.IssueTemplate{IssueNum: issueNum, IssueId: issueId} - err := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") - if err != nil { - logs.Error(err) - return - } if issueTmp.OpenEulerScore != issueTmp.NVDScore && issueTmp.NVDScore > 0 && issueTmp.OpenEulerScore > 0 { err = changeOpenEulerScoreStatus(issueTmp.CveId, 2) @@ -1012,75 +1136,59 @@ func handleIssueComment(payload models.CommentPayload) { if err != nil { logs.Error(err) } - accessToken := os.Getenv("GITEE_TOKEN") - owner := beego.AppConfig.String("gitee::owner") - path := issueTmp.Repo taskhandler.AddCommentToIssue(fmt.Sprintf(ReviewRejectScore, issueTmp.Assignee, cuAccount), issueTmp.IssueNum, owner, path, accessToken) } } else if strings.HasPrefix(cBody, cmdApe) { - issueTmp := models.IssueTemplate{IssueNum: issueNum, IssueId: issueId} - err := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") - if err != nil { - logs.Error(err) - return - } if issueTmp.Status == 3 { // The issue has been closed and cannot be operated again logs.Error("The issue has been closed and cannot be operated again,issuetmp: ", issueTmp) return } - token := os.Getenv("GITEE_TOKEN") - owner := beego.AppConfig.String("gitee::owner") - fixed := beego.AppConfig.String("labelFixed") - unfixed := beego.AppConfig.String("labelUnFix") - approveFlag := true - if issueTmp.OpenEulerScore != issueTmp.NVDScore && issueTmp.NVDScore > 0 && - issueTmp.OpAuditFlag != 1 && issueTmp.OpenEulerScore > 0 { - //Approved to modify the rating status - approveFlag = openEulerScoreReview(&issueTmp, cuAccount, owner, token) - } - if approveFlag { - mtAuditFlag := false - // Analysis command belongs to the time period - maintainerList, mainOk := models.QueryRepoAllMaintainer(issueTmp.Repo) - if mainOk && len(maintainerList) > 0 { - for _, v := range maintainerList { - if util.TrimString(v.MemberName) == cuAccount { - mtAuditFlag = true - break + if vc.OrganizationID == 2 { + comLock.Lock() + gaussMaintainerApprove(&issueTmp, cuAccount, owner, accessToken, fixed, path, vc) + comLock.Unlock() + } else { + approveFlag := true + if issueTmp.OpenEulerScore != issueTmp.NVDScore && issueTmp.NVDScore > 0 && + issueTmp.OpAuditFlag != 1 && issueTmp.OpenEulerScore > 0 { + //Approved to modify the rating status + approveFlag = openEulerScoreReview(&issueTmp, cuAccount, owner, accessToken) + } + if approveFlag { + mtAuditFlag := false + // Analysis command belongs to the time period + maintainerList, mainOk := models.QueryRepoAllMaintainer(issueTmp.Repo) + if mainOk && len(maintainerList) > 0 { + for _, v := range maintainerList { + if util.TrimString(v.MemberName) == cuAccount { + mtAuditFlag = true + break + } } } - } - if mtAuditFlag { - comLock.Lock() - maintainerApprove(&issueTmp, cuAccount, owner, token, fixed, unfixed) - comLock.Unlock() - } else { - comLock.Lock() - securityApprove(&issueTmp, cuAccount, owner, token, fixed, unfixed) - comLock.Unlock() + if mtAuditFlag { + comLock.Lock() + maintainerApprove(&issueTmp, cuAccount, owner, accessToken, fixed, unfixed) + comLock.Unlock() + } else { + comLock.Lock() + securityApprove(&issueTmp, cuAccount, owner, accessToken, fixed, unfixed) + comLock.Unlock() + } } } } else if strings.HasPrefix(cBody, cmdClose) { //close issue - issueId := payload.Issue.Id - issueTmp := models.IssueTemplate{IssueNum: issueNum, IssueId: issueId} - err := models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id") - if err != nil { - logs.Error(err) - return - } closeFlag := false if cuAccount == issueTmp.Assignee { //call the gitee api to change issue status c := strings.Trim(cBody, cmdClose) issueTmp.ErrorDescription = c - token := beego.AppConfig.String("gitee::git_token") - owner := beego.AppConfig.String("gitee::owner") // No issue closing command is provided temporarily if closeFlag { - success := CloseIssue(token, issueTmp.Repo, issueNum, owner) + success := CloseIssue(accessToken, issueTmp.Repo, issueNum, owner) if success { //update issue tpl err = models.UpdateIssueTemplate(&issueTmp, "error_description") @@ -1099,7 +1207,7 @@ func handleIssueComment(payload models.CommentPayload) { } cBody = strings.ReplaceAll(cBody, "：", ":") comLock.Lock() - analysisComment(issueNum, cuAccount, cBody, &payload) + analysisComment(owner, accessToken, path, cuAccount, cBody, &payload, issueTmp, vc.OrganizationID) comLock.Unlock() } } @@ -1109,14 +1217,8 @@ func isReviewer(path string) bool { return sr.Read("name_space") } -func analysisComment(issueNum string, cuAccount string, cBody string, payload *models.CommentPayload) { - issueId := payload.Issue.Id - issueTmp := models.IssueTemplate{IssueNum: issueNum, IssueId: issueId} - err := models.GetIssueTemplateByColName(&issueTmp, "IssueNum", "IssueId") - if err != nil { - logs.Error(err) - return - } +func analysisComment(owner, accessToken, path string, cuAccount string, cBody string, + payload *models.CommentPayload, issueTmp models.IssueTemplate, OrganizationID int8) { if issueTmp.Status == 3 { // The issue has been closed and cannot be operated again logs.Error("The issue has been closed and cannot be operated again,issuetmp: ", issueTmp) @@ -1128,12 +1230,13 @@ func analysisComment(issueNum string, cuAccount string, cBody string, payload *m logs.Error("openeuler-ci-bot, Ignore this comment") return } + if cuAccount == "opengauss-bot" { + logs.Error("opengauss-bot, Ignore this comment") + return + } } canVerfy := false issueTmp.MtAuditFlag = 1 - accessToken := os.Getenv("GITEE_TOKEN") - owner := beego.AppConfig.String("gitee::owner") - path := issueTmp.Repo //is Analyst comment and content start with '/analysis' vMap := util.ExtractCommentAnalysisAllValue(cBody) if len(vMap) > 0 { @@ -1204,7 +1307,7 @@ func analysisComment(issueNum string, cuAccount string, cBody string, payload *m } } // update gitee issue - commentUpdateIssue(issueTmp) + commentUpdateIssue(issueTmp, owner, accessToken, path) } if _, ok := vMap["issue_package"]; ok { // handle comment package @@ -1216,7 +1319,11 @@ func analysisComment(issueNum string, cuAccount string, cBody string, payload *m } if canVerfy { //Check whether the data is legal - if msg, tb, ok := checkIssueAnalysisComplete(&issueTmp); !ok { + checkFunc := checkIssueAnalysisComplete + if OrganizationID == 2 { + checkFunc = checkGaussIssueAnalysisComplete + } + if msg, tb, ok := checkFunc(&issueTmp); !ok { //send comment to issue issueTmp.IssueStatus = 1 err := models.UpdateIssueTemplate(&issueTmp, "issue_status") @@ -1232,46 +1339,57 @@ func analysisComment(issueNum string, cuAccount string, cBody string, payload *m msg = fmt.Sprintf(CommentAnalysisCplTpl, assignee, msg) taskhandler.AddCommentToIssue(msg, issueTmp.IssueNum, owner, path, accessToken) } else { - //1. change issue status - issueTmp.IssueStatus = 3 - //2. Are the cvsScore and openEuler score equal .If not equal, notify the auditor to review . - var na string - if issueTmp.OpenEulerScore != issueTmp.NVDScore && issueTmp.OpenEulerScore > 0 && - issueTmp.NVDScore > 0 && issueTmp.OpAuditFlag == 0 { - na = "\n**因OpenEulerScore与NvdScore不一致,分析内容需审核,请等待安全组审核!**" - //Notify the responsible person for review - notifyAuditorReview(payload, issueTmp) - } else { - if issueTmp.MtAuditFlag == 0 { - maintainerList, mainOk := models.QueryRepoAllMaintainer(issueTmp.Repo) - assList := []string{} - if mainOk && len(maintainerList) > 0 { - for _, v := range maintainerList { - assList = append(assList, "@"+v.MemberName+" ") - content := fmt.Sprintf("%v 仓库的CVE和安全问题的ISSUE,CVE编号: %v, "+ - "已经完成了模板填写,需要您对填写的内容进行审核,审核通过才能进行后续操作.", issueTmp.Repo, issueTmp.CveNum) - taskhandler.SendPrivateLetters(accessToken, content, v.MemberName) - } - } - assignee := "" - if len(assList) > 0 { - assignee = strings.Join(assList, ",") - } else { - assignee = "@" + issueTmp.Assignee - } - na = "\n**请确认模板分析内容的准确性与完整性, 确认无误后,请在评论区输入: /approve, 否则无法关闭当前issue.**" - cc := fmt.Sprintf(ContentReview, assignee) + tb + na - taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) - } else { - na = "\n**请确认分析内容的准确性, 确认无误后, 您可以进行后续步骤, 否则您可以继续分析.**" - cc := fmt.Sprintf(AnalysisComplete, issueTmp.Assignee) + tb + na - taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) - } + if OrganizationID == 2 { + na := "\n**请确认分析内容的准确性, 确认无误后, 您可以进行后续步骤, 否则您可以继续分析.**" + cc := fmt.Sprintf(AnalysisComplete, issueTmp.Assignee) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) // change score status err := changeOpenEulerScoreStatus(issueTmp.CveId, 3) if err != nil { logs.Error(err) } + } else { + //1. change issue status + issueTmp.IssueStatus = 3 + //2. Are the cvsScore and openEuler score equal .If not equal, notify the auditor to review . + var na string + if issueTmp.OpenEulerScore != issueTmp.NVDScore && issueTmp.OpenEulerScore > 0 && + issueTmp.NVDScore > 0 && issueTmp.OpAuditFlag == 0 { + na = "\n**因OpenEulerScore与NvdScore不一致,分析内容需审核,请等待安全组审核!**" + //Notify the responsible person for review + notifyAuditorReview(payload, issueTmp) + } else { + if issueTmp.MtAuditFlag == 0 { + maintainerList, mainOk := models.QueryRepoAllMaintainer(issueTmp.Repo) + assList := []string{} + if mainOk && len(maintainerList) > 0 { + for _, v := range maintainerList { + assList = append(assList, "@"+v.MemberName+" ") + content := fmt.Sprintf("%v 仓库的CVE和安全问题的ISSUE,CVE编号: %v, "+ + "已经完成了模板填写,需要您对填写的内容进行审核,审核通过才能进行后续操作.", issueTmp.Repo, issueTmp.CveNum) + taskhandler.SendPrivateLetters(accessToken, content, v.MemberName) + } + } + assignee := "" + if len(assList) > 0 { + assignee = strings.Join(assList, ",") + } else { + assignee = "@" + issueTmp.Assignee + } + na = "\n**请确认模板分析内容的准确性与完整性, 确认无误后,请在评论区输入: /approve, 否则无法关闭当前issue.**" + cc := fmt.Sprintf(ContentReview, assignee) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) + } else { + na = "\n**请确认分析内容的准确性, 确认无误后, 您可以进行后续步骤, 否则您可以继续分析.**" + cc := fmt.Sprintf(AnalysisComplete, issueTmp.Assignee) + tb + na + taskhandler.AddCommentToIssue(cc, issueTmp.IssueNum, owner, path, accessToken) + } + // change score status + err := changeOpenEulerScoreStatus(issueTmp.CveId, 3) + if err != nil { + logs.Error(err) + } + } } } err := models.UpdateIssueTemplate(&issueTmp, "issue_status", "mt_audit_flag") @@ -1401,6 +1519,89 @@ func checkIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok return } +func checkGaussIssueAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + msg = fmt.Sprintf("影响性分析说明没有填写或按正确格式填写") + ok = false + return + } + tbContent[0] = "已分析" + tbContent[1] = "影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + if i.OpenEulerScore == 0.0 { + msg = fmt.Sprintf("openGaussScore没有填写或正确填写(0-10)") + ok = false + return + } + tbContent[3] = "已分析" + tbContent[4] = "openGaussScore" + tbContent[5] = i.OpenEulerScore + + if i.OpenEulerVector == "" || len(i.OpenEulerVector) < 1 { + msg = fmt.Sprintf("openGaussVector没有正确填写") + ok = false + return + } + tbContent[6] = "已分析" + tbContent[7] = "openGaussVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + continue + } else { + versionfFlag = false + break + } + } else { + versionfFlag = false + break + } + } + } + if !versionfFlag { + msg = fmt.Sprintf("受影响版本排查 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + return + } + if versionfFlag { + tbContent[9] = "已分析" + tbContent[10] = "受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } else { + tbContent[9] = "待分析" + tbContent[10] = "受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "受影响版本排查" + tbContent[11] = "" + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + func checkIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { tb := `| 状态 | 需分析 | 内容 | @@ -1489,10 +1690,95 @@ func checkIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr strin return } -func commentUpdateIssue(issueTmp models.IssueTemplate) { - accessToken := os.Getenv("GITEE_TOKEN") - owner := beego.AppConfig.String("gitee::owner") - path := issueTmp.Repo +func checkGaussIssueClosedAnalysisComplete(i *models.IssueTemplate) (msg, tbStr string, ok bool) { + tb := + `| 状态 | 需分析 | 内容 | +|:--:|:--:|---------| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +|%v|%v|%v| +` + + if i == nil { + logs.Error("issue template is nil") + return msg, "", false + } + ok = true + tbContent := make([]interface{}, 12) + if util.TrimString(i.CveAnalysis) == "" || len(util.TrimString(i.CveAnalysis)) < 1 { + tbContent[0] = "待分析" + tbContent[1] = "影响性分析说明" + tbContent[2] = fmt.Sprintf("影响性分析说明没有填写或按正确格式填写") + msg = fmt.Sprintf("影响性分析说明没有填写或按正确格式填写") + ok = false + } else { + tbContent[0] = "已分析" + tbContent[1] = "影响性分析说明" + tbContent[2] = util.TrimStringNR(i.CveAnalysis) + } + if i.OpenEulerScore == 0.0 { + tbContent[3] = "待分析" + tbContent[4] = "openGaussScore" + tbContent[5] = fmt.Sprintf("openGaussScore没有填写或正确填写(0-10)") + msg = fmt.Sprintf("openGaussScore没有填写或正确填写(0-10)") + ok = false + } else { + tbContent[3] = "已分析" + tbContent[4] = "openGaussScore" + tbContent[5] = i.OpenEulerScore + } + if util.TrimString(i.OpenEulerVector) == "" || len(util.TrimString(i.OpenEulerVector)) < 1 { + tbContent[6] = "待分析" + tbContent[7] = "openGaussVector" + tbContent[8] = fmt.Sprintf("openGaussVector没有正确填写") + msg = fmt.Sprintf("openGaussVector没有正确填写") + ok = false + } else { + tbContent[6] = "已分析" + tbContent[7] = "openGaussVector" + tbContent[8] = util.TrimStringNR(i.OpenEulerVector) + } + if i.AffectedVersion != "" { + versionfFlag := true + affectedVersionArry := strings.Split(i.AffectedVersion, ",") + if len(affectedVersionArry) > 0 { + for _, affect := range affectedVersionArry { + versionArry := strings.Split(affect, ":") + if len(versionArry) > 1 { + if versionArry[1] == "受影响" || versionArry[1] == "不受影响" { + continue + } else { + versionfFlag = false + break + } + } else { + versionfFlag = false + break + } + } + } + if !versionfFlag { + tbContent[9] = "待分析" + tbContent[10] = "受影响版本排查" + tbContent[11] = fmt.Sprintf("受影响版本排查 没有分析或未按正确格式填写:%v", i.AffectedVersion) + msg = fmt.Sprintf("受影响版本排查 没有分析或未按正确格式填写:%v", i.AffectedVersion) + ok = false + } else { + tbContent[9] = "已分析" + tbContent[10] = "受影响版本排查" + tbContent[11] = util.TrimStringNR(i.AffectedVersion) + } + } else { + tbContent[9] = "已分析" + tbContent[10] = "受影响版本排查" + tbContent[11] = "" + } + tbStr = fmt.Sprintf(tb, tbContent...) + return +} + +func commentUpdateIssue(issueTmp models.IssueTemplate, owner, accessToken, path string) { if accessToken != "" && owner != "" && path != "" { cvlnCenter := models.VulnCenter{} err := models.GetVulnCenterByCVEID(&cvlnCenter, issueTmp.CveId) diff --git a/controllers/upload.go b/controllers/upload.go index 004ed13..d82c195 100644 --- a/controllers/upload.go +++ b/controllers/upload.go @@ -252,6 +252,13 @@ func (u *UserUploadController) Post() { orCve.IsExit = 1 logs.Info("对应src-openEuler的数据为: ", gits) break + } else { + opy := models.OpenGussYaml{PackageName: pkList[0], Version: pkList[1]} + openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + if openErr == nil && opy.Id > 0 { + orCve.IsExit = 1 + break + } } } } diff --git a/cve-py/common/files.py b/cve-py/common/files.py new file mode 100644 index 0000000..e63ac4a --- /dev/null +++ b/cve-py/common/files.py @@ -0,0 +1,54 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +common files + +Authors: zhangjianjun +Date: 04/01/2021 11:01 AM +""" + +import os +import shutil + + +def mkdirs(file_path, file_flag): + """ + Create a directory, return the creation success or failure, bool value + file_flag ==1 Delete all files + return bool + """ + path = file_path.strip() + path = path.rstrip("/") + isExists = os.path.exists(path) + if not isExists: + try: + os.makedirs(path) + except OSError: + return False + else: + if file_flag == 1: + del_list = os.listdir(path) + for f in del_list: + file_path = os.path.join(path, f) + if os.path.isfile(file_path): + os.remove(file_path) + elif os.path.isdir(file_path): + shutil.rmtree(file_path) + return True + + +def file_isexists(file_path): + """ + Determine whether the file exists + return bool + """ + if os.path.exists(file_path): + return True + else: + return False diff --git a/cve-py/common/times.py b/cve-py/common/times.py new file mode 100644 index 0000000..cdcf6b1 --- /dev/null +++ b/cve-py/common/times.py @@ -0,0 +1,24 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +common time + +Authors: zhangjianjun +Date: 04/01/2021 11:01 AM +""" + +import time + + +def get_current_time(): + """ + get current time + """ + now = str(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())) + return now diff --git a/cve-py/controller/taskcontroller.py b/cve-py/controller/taskcontroller.py index e4e00f0..959f13f 100644 --- a/cve-py/controller/taskcontroller.py +++ b/cve-py/controller/taskcontroller.py @@ -13,8 +13,9 @@ Authors: xiaojianghui Date: 10/22/2020 11:01 AM """ -from tabletask import (runtask, mappingtask, toexcel, export_excel_task, import_excel_task, package_committer_task, - spec_error_task, repeattask) +from tabletask import (runtask, mappingtask, toexcel, export_excel_task, + import_excel_task, package_committer_task, + spec_error_task, repeattask, gauss_yaml, issue_statistics) from gitwebtask import genegroup, yamltask from emailtask import sendingtask, issue_record_email from deletetask import deletefiletask @@ -121,3 +122,24 @@ def run_package_committer_task(): print('the task of parsing package_committer_excels start') package_committer_task.import_data() print('the task of parsing package_committer_excels complete') + + +def parse_opengauss_yaml_task(): + """ + Download the yaml file first, then parse the yaml file of opengauss + Returns:None + + """ + print('the task of Process opengauss yaml data start') + gauss_yaml.proc_gauss_yaml() + print('the task of Process opengauss yaml data complete') + + +def issue_statistics_email_task(): + """ + Parse the issue statistics recipient list + return None + """ + print('Parse the issue statistics recipient list start') + issue_statistics.get_issue_excel() + print('Parse the issue statistics recipient list complete') diff --git a/cve-py/controller/timertaskcontroller.py b/cve-py/controller/timertaskcontroller.py index a725721..1daf9f8 100644 --- a/cve-py/controller/timertaskcontroller.py +++ b/cve-py/controller/timertaskcontroller.py @@ -41,6 +41,10 @@ def timertask(): scheduler.add_job(taskcontroller.run_whiltlist_task, 'interval', hours=2) # parsing the table package_committer_excels scheduler.add_job(taskcontroller.run_package_committer_task, 'interval', hours=1) + # Parse opengauss yaml file + scheduler.add_job(taskcontroller.parse_opengauss_yaml_task, 'cron', day_of_week='0-6', hour=8, minute=30) + # Parse the issue statistics recipient list + scheduler.add_job(taskcontroller.issue_statistics_email_task, 'cron', day_of_week='0-6', hour=5, minute=30) scheduler.start() except SystemExit as err: print("Err:", err) diff --git a/cve-py/downloadtask/downloadfiletask.py b/cve-py/downloadtask/downloadfiletask.py index f30c229..4921ed5 100644 --- a/cve-py/downloadtask/downloadfiletask.py +++ b/cve-py/downloadtask/downloadfiletask.py @@ -193,3 +193,56 @@ def download_spec_error_excels(): if os.path.exists("./spec_error_excels/" + name): print("Download the package_committer table successfully:" + name) i += 2 + + +def download_gauss_yaml(file_path, file_url): + """ + Download yaml data from gitee + """ + try: + r = requests.get(file_url) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + with open(file_path, "wb") as code: + code.write(r.content) + + +def download_excel(file_dir): + """ + download excel + """ + local_dir = "./" + file_dir + gitee_dir = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/' + file_dir + urlx = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/%s/" %(file_dir) + if os.listdir(local_dir): + shutil.rmtree(local_dir) + os.mkdir(local_dir) + try: + r = requests.get(gitee_dir) + except requests.exceptions.ConnectionError: + print('ConnectionError') + return + except requests.exceptions.ChunkedEncodingError: + print('ChunkedEncodingError') + return + html = etree.HTML(r.content) + if len(html): + i = 3 + while True: + try: + name = html.xpath("//div[@id='tree-slider']/div[{}]/div[1]/a/@title".format(i))[0] + except IndexError: + break + if name.endswith(".xls") or name.endswith(".xlsx"): + url = urlx + name + r = requests.get(url) + print(r.status_code) + with open(local_dir + "/" + name, 'wb') as code: + code.write(r.content) + if os.path.exists(local_dir + "/" + name): + print("Download the excel successfully:" + name) + i += 2 \ No newline at end of file diff --git a/cve-py/gauss_yaml/gauss_yaml.yaml b/cve-py/gauss_yaml/gauss_yaml.yaml new file mode 100644 index 0000000..16fcbc0 --- /dev/null +++ b/cve-py/gauss_yaml/gauss_yaml.yaml @@ -0,0 +1,252 @@ +jemalloc: + cpeName: jemalloc + version: 5.2.1 + url: 'https://github.com/jemalloc' +libedit: + cpeName: libedit + version: 3.1 + url: 'http://thrysoee.dk/editline/' +OpenSSL: + cpeName: OpenSSL + version: 1.1.1d + url: 'https://www.openssl.org/news/vulnerabilities.html' +lz4: + cpeName: lz4 + version: v1.9.2 + url: 'https://github.com/lz4' +Postgresql JDBC Driver: + cpeName: Postgresql JDBC Driver + version: 42.2.5 + url: 'https://jdbc.postgresql.org/index.html' +Postgres-XC: + cpeName: Postgres-XC + version: 1.1 + url: '' +PSQL ODBC: + cpeName: psqlodbc + version: 10.03.0000 + url: 'https://www.postgresql.org/' +snappy: + cpeName: snappy + version: 1.1.7 + url: '' +protobuf: + cpeName: protobuf + version: 3.7.1 + url: 'https://github.com/protocolbuffers/protobuf' +masstree: + cpeName: masstree + version: v0.9.0 + url: '' +mysql_fdw: + cpeName: mysql_fdw + version: v2.5.3 + url: 'https://github.com/EnterpriseDB/mysql_fdw' +oracle_fdw: + cpeName: oracle_fdw + version: v2.2.0 + url: 'https://github.com/laurenz/oracle_fdw' +llvm: + cpeName: llvm + version: 7.0.0 + url: 'https://github.com/llvm' +ncurses: + cpeName: ncurses + version: 6.2 + url: '' +paramiko: + cpeName: paramiko + version: 2.6.0 + url: 'https://github.com/paramiko/paramiko' +huaweicloud-sdk-c-obs: + cpeName: huaweicloud-sdk-c-obs + version: 3.1.3 + url: 'https://github.com/huaweicloud/huaweicloud-sdk-c-obs' +Kerberos 5 (KRB5): + cpeName: Kerberos 5 (KRB5) + version: 1.17.1 + url: 'https://kerberos.org/' +Curl and Libcurl: + cpeName: Curl and Libcurl + version: 7.68.0 + url: 'https://curl.se/docs/security.html' +orc: + cpeName: orc + version: 1.5.7 + url: 'https://orc.apache.org/security/' +libiconv: + cpeName: libiconv + version: 1.16 + url: 'https://www.gnu.org/software/libiconv/' +pcre: + cpeName: pcre + version: 8.42 + url: 'https://www.pcre.org/' +psutil: + cpeName: psutil + version: 5.6.7 + url: 'https://pypi.org/project/psutil/' +Netifaces: + cpeName: Netifaces + version: 0.10.9 + url: 'https://pypi.org/project/netifaces/' +nghttp2: + cpeName: nghttp2 + version: 1.39.2 + url: 'https://github.com/nghttp2/nghttp2' +'python::cryptography': + cpeName: python::cryptography + version: 2.7 + url: 'https://pypi.org/project/cryptography/' +'python::cffi': + cpeName: python::cffi + version: 1.12.3 + url: 'https://pypi.org/project/cffi/' +pycparser: + cpeName: pycparser + version: 2.19 + url: 'https://pypi.org/project/pycparser/' +'python::asn1crypto': + cpeName: python::asn1crypto + version: 1.2.0 + url: 'https://pypi.org/project/asn1crypto/' +idna: + cpeName: idna + version: 2.8 + url: 'https://pypi.org/project/idna/' +six: + cpeName: six + version: 1.12.0 + url: 'https://pypi.org/project/six/' +ipaddress: + cpeName: ipaddress + version: 1.0.22 + url: 'https://pypi.org/project/ipaddress/' +pyasn1: + cpeName: pyasn1 + version: 0.4.7 + url: 'https://pypi.org/project/pyasn1/' +PyNaCl: + cpeName: PyNaCl + version: 1.3.0 + url: 'https://pypi.org/project/PyNaCl/' +bcrypt: + cpeName: bcrypt + version: 3.1.7 + url: 'https://pypi.org/project/bcrypt/' +zlib: + cpeName: zlib + version: 1.2.11 + url: 'https://github.com/madler/zlib' +postgresql-hll: + cpeName: postgresql-hll + version: 2.10.2 + url: 'https://github.com/citusdata/postgresql-hll' +numactl: + cpeName: numactl + version: 2.0.13 + url: 'https://github.com/numactl/numactl' +pyOpenSSL: + cpeName: pyOpenSSL + version: 19.0.0 + url: 'https://pypi.org/project/pyOpenSSL/' +Apache Arrow: + cpeName: Apache Arrow + version: v0.11.1 + url: 'https://github.com/apache/arrow' +Apache Thrift: + cpeName: Apache Thrift + version: 0.13.0 + url: 'https://github.com/apache/thrift' +boost: + cpeName: boost + version: 1.71.0 + url: 'https://www.boost.org/' +brotli: + cpeName: brotli + version: v0.6.0 + url: 'https://github.com/google/brotli' +c-ares: + cpeName: c-ares + version: 1.15.0 + url: 'https://c-ares.haxx.se/vulns.html' +double-conversion: + cpeName: double-conversion + version: 3.1.1 + url: 'https://github.com/google/double-conversion' +flatbuffers: + cpeName: flatbuffers + version: 1.11.0 + url: 'https://github.com/google/flatbuffers' +google/glog: + cpeName: google/glog + version: 0.4.0 + url: 'https://github.com/google/glog' +grpc: + cpeName: grpc + version: 1.22.0 + url: 'https://github.com/grpc/grpc' +libcgroup: + cpeName: libcgroup + version: 0.41-21.el7 + url: 'https://github.com/matsumotory/libcgroup' +libxml2: + cpeName: libxml2 + version: 2.9.9 + url: 'https://github.com/GNOME/libxml2' +rapidjson: + cpeName: rapidjson + version: 4b3d7c2f42142f10b888e580c515f60ca98e2ee9 + url: 'http://rapidjson.org/' +Zstandard: + cpeName: Zstandard + version: 1.4.4 + url: 'https://github.com/facebook/zstd' +cJSON: + cpeName: cJSON + version: 1.7.11 + url: 'https://github.com/DaveGamble/cJSON' +libevent: + cpeName: libevent + version: 2.1.11 + url: 'https://github.com/libevent/libevent' +p7zip: + cpeName: p7zip + version: 16.02 + url: 'https://www.7-zip.org/' +gcc: + cpeName: gcc + version: 8.2.0 + url: 'https://gcc.gnu.org/' +gmp: + cpeName: gmp + version: 6.1.2 + url: 'https://gmplib.org/' +mpc: + cpeName: mpc + version: 1.1.0 + url: 'http://ftp.gnu.org/' +mpfr: + cpeName: mpfr + version: 4.0.2 + url: 'https://www.mpfr.org/' +bounds_checking_function: + cpeName: bounds_checking_function + version: v1.1.10 + url: 'https://gitee.com/openeuler' +isl: + cpeName: isl + version: 0.18 + url: 'https://gcc.gnu.org/' +AdoptOpenJDK: + cpeName: AdoptOpenJDK + version: jdk8u222-b10 + url: 'https://adoptopenjdk.net/' +Apache Maven: + cpeName: Apache Maven + version: 3.6.3 + url: 'https://maven.apache.org/security.html' +unixODBC: + cpeName: unixODBC + version: 2.3.6 + url: 'http://www.unixodbc.org/' \ No newline at end of file diff --git a/cve-py/issue_statistics/to_email.xlsx b/cve-py/issue_statistics/to_email.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..1e3a906e8a8abf378d91f441fdb12b5cc348e0fd GIT binary patch literal 10612 zcmeHtWl)^kvNrA>2*HB8%MdiU+n~WExVr^s7~CzmLvRQX+zAfBg1ZC>?gSFJkes`d z&DrPF{rgpYYpUj*nQEE!bg%BN)eV$^hCzaOSV8h@f)CICU5NJ|jBE{o4z_lV%)omx z?E4=Oev*AdcE96=gn+Pxf`GvIlgz-*j>*m1Dm`{s8k!XweCdCPQB|=M6&#<8Ez}*M zUfCFx%xkSrh1f8ieXg-kkKqaZsx`tt%D@E5nXb{^HP(^KlbRy6Y&R5uG0K&CI@S@* zUO9);!{q2)%|%S;7?@|3O=n8FMIiVx?#5%5j|<%og(M8;sbeeIi83j$B1_E;!1=-zhi|lTZ#(B0p-no;AZi`Dm>}Tw<=4D)y zQ?I0pPej-lJ2R^(k`U6GtmA9ZJCcA%Qyo}Hnepx zen2`RMn|EG1Uv9h?mIf_C1WLiU36l1)ZPFk)hDxNa=ThgWJ+I`o6D;5681N%lc_)lAS3ow_5oSU_IJH{odAW#;S7DxkRTyB^9COCcX6TT^-@b@Z62+ATNi6dm zDT?&P_QO%>MAWkK1lmVyv5;YhRMKkKEd?h$L&_{%eMt;9F*{aRkB^~kO3A=efLgk|{dW8v z`R^E~$}Cge-DB%}kMoniVeIVaWNZD?pQHP2KrGl|mkHPCiI+Vh(ge|@bkSCgX8C!u z#G`FlWe^TG+B;gN6*hh9l>)3qeVV_uhE*0)9`(|B*U~NZ!=IjTFk>SU@ zJhUWChp6rikYeBM+DgLPk=3@oPr+g70?(fkILq9k>IqhKj^Z5UO6?X%NrldEmS9rW z2+z`a2pY*Dl&H51KCjEfm>FaT3D(i6`OFDOSiJ}V%5}{U<%ex^Z{&KNb|i%7p(v(& zM13Nyb{4tb7QId!6uTa_;M{of6R&@c5zzN|xmo>;4*yJm_KU#P*1^)z%-GoJaU?kr zx4=lackT#S2#9C@M*EY6^{1c5&)dwipadSm?-8LaPFJOuX@O>R&}r24p+lX6EV~P= z0aba@4rymQ68#}>Yw<=2M99jb1ZYFUPUWa`0M7;!hNSSRDWEipLkFp-+q{?3JgH)w zWz}S95V>dlDYv?BT*HKt6RaV%17OS@$9dju#m`lNBWKmkqqmC^c9Ocv%nK*s)N#<9 z{Y|7CqUMshyFtq%Accc*EL^VDp69n#ZV7pX$EjcL!4HpBAu;k% zMxc3o`gokC!~*wiac*wSv_{5QwW+(y7y8kd*Um$wZ@m2VBl;N|R%{5y7m=YZqu5bI z_{<32I=oxU65OFiIw-D_cb*<%RbuEwUBJ_1po>&Jdqs>_=;#phiP;ope>G~E5YD4A zl2B_2qX$_>ufa;i)u*&UaMJQ*5NqpTP+|O@#Ui+J1VbNiT0xR-stz6}U>*&f zV>p-=tID#Jty%cSgzmMok=vJ=Rd|c=SxclHFXB)Yj-M^7bB$ zQ}f67*!yarj_Zobs);w#uW06mxNCzO%2-ZAwHRkOspTp-(~2ZAyv#i~AuiGT^+shN zkkTL!QXv!=hm$AvOMYbV>#r#nz4?`9rFtQIi`l7R;33wUbkkb~lKIw3e zmg6z1aeKQ(91t^bl5DrG*VSID3b1k*VMk360!-+oPKqU8*lmfaa?r`!H@?}-X8iUJC#w7NZIM0-HEZgD zN0eI>S*fo{l||V*Opsr`Mj7SH=>dQ>jkg) z^^P$@yk6$nkDl3_HEm7tQQ$-dB^@Vmw9v*ZPmo6ut7)xKWTGNLfa9LU_Bnu#F*NJCe9`Mc-V&+mv znO-Kwq^wEJ+L!~GN+3686B56ULAmYEQY%YfPH3s86uWJE={!Oglub8P1vn)M<1k-GP6nW zPqxPy#QD_1UNd&qHEcr(LgT4%h+CvYV_uLZ&2!-+gaV+O{|VC6Li^HSCa;w=Rylb*7N$&Ppr2f)+X@n zWU^~y;J7OI?z{#wElq@zrvi;CIM&79q(DAQ!+$2Leb&yCv$I!mr4Rmscj0Vv7E(X8~c2k8|;*YTy&z-5Dq2X~6tfojhCM8tB zEg00Jb+7p4D57b11akspxR_H`mFDo#TiUAL4L80Fni(K{KesBv+nf_ry)G`_BZ-s5 zZrx)h_JoVJIsuthadclUN%aGedQ%OMlbo|{H>f&A#ku9c;$DuAvN;v-2~J1=Mokwy z(Po0WMKsh0oL{`AaSS`|)M%6pbdyvfHgDPTQUS;hyf&7{EDJ(~^K#+qv4FKfmAhc9 zPyZ+-=JEk{kL|D};%yZ|hC5;*k(|U++_<>z?W`TZ*Fk6WZ(QbY$hqjc4C8&L^KGbD zkyqyWUv&olc;9bT<+(X5UXYXuqgwx?(j`-QrU*JD+5&lSG%zq(TEw?IB`(3I0nLP; z9#*<|tUpg(1Ehb|n+L($SWcOXd8|4%3&ynOy-7(_dZ{RS*0~CaUl}sfwc<@woHY_H zU4P7iBh=P@o>p6%>=5Rd;?ym}3aVaDq9m0cER!E&h+|y9&eMiFRl`7cqEA?)aebpI zJCGSe*UfU9S(J16q~2d0&jS6*PKtqLJbC=A9!MHQ7a#EX5Xp>CcHh^`k6t$^Fw4Kh z;*_zk^|qGV)I6ysFfMN{7ZMBmf*c1)*d?1Gd7C#-9kAHMAeD&4hN+72ib`z{a+bQZ z!g8f}X`UZjxG>csHW3oUi|=#bks<^Ve8JuA=}4#1dO1gU)x=AjI+PW!6(lDTH!Fb4 zYKv~tRIFp0XzF3Bh{?}tS#6=A22tIqHYz!Biz>$TMa`0aDy?VmT$bx-t+|IK>4hkw zHU;Sq^a}wxPjT62yQm*VF0*RMu6_tV0af}HzBDC3&qK~5A{1OVg12TdSDLCZ-_0va zw4O3u*xB@qdZs1gg5{GrKI87VS>@<(%n*^TIrIJlk(Qao(cTy~I9t&rnl+MDAPL83 zXbm*KvdwrktKS)4Q-f-@uJHGwCO4^>5A{Ch(aJ$Up#Dm7j!y1Y#*Pn_%%rwQ)Vw;* z9dXE^euJ+ICVL36dwsfWT^yc_B(i3am~f6+yi^*VOe8;-JOEml&S|hchb)^3TaEtx zUU9s@HU#qP5@W4a*|<-h$e+BfUb0O~6{K=~5>1(j9O186=Du7AYO()Oh->EBTz@EG zAo5~ruyGT-U$`E0gZjgxZs9{?`FWh$j5)00HsH*#qC#)^``1^T*ZuY@i+o4Tq9CeH zZxI2>*shX%r8cjXbIl*;+oQ;y zb(-nB%T2`D^n9qE6td;U?zQtmyH=C!*L)Q=MHI)UtWH#U{n0A zcP?EGqL4jWaGxglH6!GaukCR{<>YA}QP+fCo98XYvh!%@jW4&b+3YsHIAKfmR*+&i zQz8QTiI4Arl3m#4C4S>>;_8Sq=A=sEm)U2X;fL2JPIbUMBrzJa}7stsk;b1?0+Lu8n5(M(V$SG zu<_@|#hg)g8|H?iusIi-w-^4XQT|03{zb3sURb}V9X_gmO4!R&Bd+|4B9Zv%trRBe zqzQ91`qPMd#G!@mz)P`Z(n@v56Cx(uYhdz~Q2PLIQ1}Cq_Ur?XT+2?=1X12A{Eo)qU_S$v+xFY<)quU9Eowyy~=L1RpBNB0*nK zo{mt;tmGK@LB>)+Pk=}KW8zNyPnu`7S(rnWf^sRADP+EF5r=;(?$@5P;t4ybQ*TR_ zS?MTUP&y5Z);nbct8Z+)HpxaBcqJFOkj_Z`&WJ*V^qV-Xa}mUjr?uQSWStA8E97hD z?ptBgn!Q-_*=rxEyLBNd6wAw;dgw4dd>uV|eNibUYa1Bd7k(&DZ*JhXc|*_?Uxvj} zeC-hGwcVF6p!e-kl8*6K$O-_Et3S8c=C4>sf733P@|HYTvD1(ztfow!V+NUT>glkI zCstO#Mt10sENHSvb3_*EsWd7F`>?UI9NJOUv-kLu@d~O%9O=ULiX*fzNR01*Ln6J5 zPrQKryd{xP-BvtEFzjDxfJ4RPG$b%8NKg8GODlv1=qT_&O-?tvA&}t+26PH)PF4P& z#2sZ^hMA`KLNzG8lb;;2o~tK7p60q0pjV+v3st{^@2_X445G!Z?vUyeK%$m6V3m?c zu_1M@Yayu*IoY{vVgSw4fQg;K&gSY35@G3P`GiEBqLlm;J1J>aHZ@Q_0(UfX*oz>L=j_jzKaRK`Hl%R<4`#7(CyxTs^ zp{QWV$G14Dl@M4dn(YCGV=$th57FLu=A4VTC1D({j>xcx7};zGVw)UK;Vrs$lA`>)LI@J3VW2RE zx}COcIHJ9Ndwp4wFuuP>eg-r}#>x`kDI%X&I|fY%f{wJ1D0V0z7enDYa0B$riaL-M z+{cN`OM*+*RLK?^$Y6DW2@arPHdzT4F82)wJJ(_W%v986O_7B=L3Xe>N}?19+Ojf8 z!cSe0B#$`p)p3@h3$f`6qWvk$R58XGtl+TLdg$)o7a~kX--h+a*5Asjt%JdCryr(P zKja?lGWCHb#@{RG@^%-8)Lo_mH$vZnXi-C7%PWg$dH0wPolJiKSYLH-ncmgExc?op;2!lpJ{$TqeO8<~xxR#mQcqld zK=*cT+-cNZR!3R9WFVINiM94}^*bND+Y5e#_^}`#!vHqV?bN3|WC3=D6l?+d4Z_)5 zvvCUT9Jx&jJv03*#WeP!2_P83gg66eM7@q^ELQflugTC5Chgdw$xHgpa@?pLR+%|? z1F5D&c)TpV`6L4y!)c+GFy&BI9f};Fs_Fi%q9R#5K9ZKujkA#od16*9Jf2DdX(6`j zje~84s33Ij7tKDlXJfeGYU1b!rCGy6tmdr-_sVEK?sR?I1HZ-Z@+)3eitqph%qEaE6K)) z%tV<8D?FWhT)#$>x84u4ExI>@PA0OD!9eXUU z3&VEMCTs4-Zdj;HB@DLL$ngE7$-xwy50n^#7{nL_g`&>9t%kQ1@1fq?R_Ti|M_&pL z*U-WPBRN0Q^b=N+FdvoWc4KEVvCh|EpJ2YHD=kA20`oAG&9Z44NE{ds4W!15X)5w? zD-+#wP9eFl5jP5&XV$D7BQ{4ROK9 z_hDROfJQDkWO}2F0nfSHxQ2~603=E_9JfDr(P1KsQox)r*~OnUu%o?m)zu+|BoA#! zWeVWB>0Xw%JtclI&9|dR4`u#)=7~=-A$d@wBG8Izwa0ER!B(j zey`m3y$keT`+ewfytcKq{CVdWqa)MBa^IGNy%Xtck(n-I>z(r}tS7)LJmukhXQ|#Q z$;Nxv@{ZP})24ago144aNb2Nu9?mx?;yeIlx;&e7`)4)DYUZ7r`H5lwSd0ctZ4GG3 zAAy3?ljoDg)wf6H`+g+t0kO5Ljg*GjHbSQ?@6k|f@OJ@GGuQmcpP+HGVS_c_^9|Z0 zZ;p@pRi51$%oM}`_-3vPJ3d0L2^|Mth|ud~O%}%>xpavI@LqNTu3yqhW1}A{_95it zv0GWgZwU#72sGa0s?@in8gm;@x@}AuPUtnoje<{`ZKbFqDi?#W-!q!Gsv9M(vUtEX z*pm08;GULly%z|_hN&8tk#+bs7sTHE@x~P-UU?xC0$vU_SEvVW`FvK7`nqu^9AO$| zRVs1L&>%x!qvEJ)%fFLp^!g?92^p=oPs=sr1K9t3^LBr;2BGI*Z1uR2~WJo0o?U-XwOZJC62SF!7aY5IEd`X&Ih z$rnGzL=mIosWi$K#7i4zKDMI*6wKrWh`r+$FeC?Yw%7n|22R9#GM@wG__~+qf<2nx z?76_r-)wD*cR#K0C+CWcu+wj-#H2f0MdO5nJacE-FwVu`;J8N|oL>$ex=II?mYFc3 zifMzBltUN3Xt&ujB=>5|6Nge}4Sy4ynAqy95qhRa(xRL1o5}+sP zaKc#P`k%4<*DdBh!AWv;fOm5bL+~CBlbPTi(N85d2g7$80?gGn<8$_Ka;;?We;4C)1-vPb=ToTEabK|6T zL`*dXfICZ@rdrwZ>ux1odZ>LSDPPr}A@sgR{>pK-_Pwo^d623XH`B*u`zjP%FbBu5 z#NGFXqLWjl56)y?GFV72$XydBFi1_;G3VJX-X`_ebZPM8uvq{^3M)qGJ@Q=3VKY{#ZA$Q`-J-FxhfrR`Q z$p6Oiy9N3S?U;`H$qJ0XpT2xx;)+g|Kl=ul4bQAZtluXeSJQ7KSJimFGb~mK9F6NW z*u1+%H)+xE$32X%3M#4hUMsy6HI0NG8*6|sngI*YcjzIx$HeK5b z4kB`n^Zkt&Jl$|otn7@x%xDFrq2%|fqRrH}>q?VEG!!5x6>6zv@kCN|1Yvpe5gW_w zW^;R{D(WmIarNpuJ8tTBu??gl>ZuVTuL`xmD^O!;O^mr?AR+r^l{iUwc&MOt>)s%g zATimHgl3SyH>$L+*G=>I(l|n08UNUunA5PF@t%u+Q!HlwX{DD@c+RWXeEGaQrm>A? zVN~L-)9?Fa@4nB?%CSAMH#ATN5(@igs{b_-^w>)GhJt{EAh~~!C;La6{Sxq>gZzVl zU(-d2&wq#fzbkzl^dC5WDQ*0V=jXBcZ^|B`;IH*C%Ji`Qn0orVzQ=P<55&LpQUA&I zuPLa<5Ir8|`OO`D@7{kY{om1^$Lx>iRDQFc-RHdD?0-$I{I^ZthxLE0hfPz!{A-Nk zVY~lqlE0(0M*K%gc>G5n=Z*(OzfeLX`Gf7>GRb2(k88@`a<~b9%K5hn^Ra}-x0$~s zKw|xp@Ymhue*^39dnddvFaB!kJ=XZR`}bR;$e(P#dx4JyJT5GM3!o+XsrOIV{@b5^ zn7sM59xN>O%ff%8m;c855PW~F2VOqXzZCy#hUb5i+22pc<8<~Q;g{WQ{)2=^$?Y)~ zkB5VP3$*{Y?fe$?Sjyu*{ckCJ@Q*k4Q_9aNjsLq8hI{w8_lG~A{VC;V?;j`w`%th# Q;M~8c?nj)sC?3B34_V?J*#H0l literal 0 HcmV?d00001 diff --git a/cve-py/issue_statistics/~$to_email.xlsx b/cve-py/issue_statistics/~$to_email.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..1e0c8b3a64122aeea8ded50bd91585a6113ab2a6 GIT binary patch literal 165 ycmd;eOv%m6%PcM_N-W7QQXm%aGB`4%Fyu01GUNeqF+&MM5kn%7%m>m6REGf9mK7ZU literal 0 HcmV?d00001 diff --git a/cve-py/main.py b/cve-py/main.py index 73fa067..f988f87 100644 --- a/cve-py/main.py +++ b/cve-py/main.py @@ -17,8 +17,6 @@ Date: 10/22/2020 11:01 AM """ from controller import timertaskcontroller - - if __name__ == '__main__': print("The program starts, waiting for the timing task to execute") timertaskcontroller.timertask() diff --git a/cve-py/tabletask/gauss_yaml.py b/cve-py/tabletask/gauss_yaml.py new file mode 100644 index 0000000..c4f5a5a --- /dev/null +++ b/cve-py/tabletask/gauss_yaml.py @@ -0,0 +1,238 @@ +#!user/bin/python3 +# -*- coding:UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" +Handle failed tasks + +Authors: zhangjianjun +Date: 04/01/2021 11:01 AM +""" + +from dbConnecttion.MysqlConn import Mysql +import pymysql +from common import files +from common import times +import yaml +from downloadtask import downloadfiletask + + +def download_gauss_yaml(): + """ + Download the yaml file of opengauss on gitee + return file_name + """ + file_path = "./gauss_yaml/" + mk_ok = files.mkdirs(file_path, 1) + file_name = file_path + 'gauss_yaml.yaml' + if mk_ok: + # download the yaml file + file_url = 'https://gitee.com/opengauss/openGauss-third_party/raw/master/Third_Party_Open_Source_Software_List.yaml' + downloadfiletask.download_gauss_yaml(file_name, file_url) + return file_name + + +def parse_yaml(file_name): + """ + Parse yaml + return dict + """ + with open(file_name) as f: + yaml_data = yaml.safe_load(f) + return yaml_data + + +def add_package_id(pk_id): + """ + add index + """ + packg_id = 10000000 + if pk_id > 0: + packg_id = pk_id + 1 + return packg_id + + +def store_yaml_data(yaml_data): + """ + parse dict + Store yaml data to mysql + return None + """ + mysql = Mysql() + if yaml_data != None and len(yaml_data) > 0: + for yaml_key, yaml_value in yaml_data.items(): + try: + origin_data = select_yaml_origin_data(yaml_key, yaml_value, mysql) + if origin_data != None: + update_yaml_origin_data(origin_data["id"], yaml_value, mysql) + else: + insert_yaml_origin_data(yaml_key, yaml_value, mysql) + packages_data = select_yaml_data(yaml_key, yaml_value, mysql) + if packages_data != None: + update_flag = False + for pk in packages_data: + if pk["package_id"] >= 10000000: + update_flag = True + update_yaml_data(pk["package_id"], yaml_key, yaml_value, mysql) + update_yaml_detail_data(pk["git_id"], yaml_key, yaml_value, mysql) + mysql.end() + break + if not update_flag: + package_ids = select_yaml_lastdata() + if package_ids != None and package_ids["package_id"] >= 10000000: + packg_id = add_package_id(package_ids["package_id"]) + else: + packg_id = add_package_id(0) + last_id = insert_yaml_data(packg_id, yaml_key, yaml_value, mysql) + insert_yaml_detail_data(last_id, yaml_key, yaml_value, mysql) + mysql.end() + else: + package_ids = select_yaml_lastdata() + if package_ids != None and package_ids["package_id"] >= 10000000: + packg_id = add_package_id(package_ids["package_id"]) + else: + packg_id = add_package_id(0) + last_id = insert_yaml_data(packg_id, yaml_key, yaml_value, mysql) + insert_yaml_detail_data(last_id, yaml_key, yaml_value, mysql) + mysql.end() + except pymysql.err.IntegrityError: + print(pymysql.err.IntegrityError) + mysql.dispose(2) + mysql.close() + + +def insert_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + insert data + """ + insert_sql = "insert into cve_git_open_euler(package_id,package_name," \ + "version,origin_url,create_time,update_time,cpe_packname) " \ + "values(%s,%s,%s,%s,%s,%s,%s)" + val = (package_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), yaml_value["cpeName"]) + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def insert_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + insert data details + """ + insert_sql = "insert into cve_git_package_info(git_id,package_name," \ + "version,origin_url,create_time,update_time,decription) " \ + "values(%s,%s,%s,%s,%s,%s,%s)" + val = (pg_detail_id, yaml_key, yaml_value["version"], + yaml_value["url"], times.get_current_time(), + times.get_current_time(), "") + last_id = mysql.insertOne(insert_sql, val) + return last_id + + +def update_yaml_data(package_id, yaml_key, yaml_value, mysql): + """ + update data + """ + update_sql = "update cve_git_open_euler set origin_url = %s,update_time = %s, " \ + "cpe_packname = %s where package_id = %s and package_name =%s and version = %s" + val = (yaml_value["url"], times.get_current_time(), yaml_value["cpeName"], + package_id, yaml_key, yaml_value["version"]) + mysql.update(update_sql, val) + + +def update_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): + """ + Update data details + """ + update_sql = "update cve_git_package_info set package_name = %s,version = %s, " \ + "origin_url = %s, update_time = %s where git_id = %s" + val = (yaml_key, yaml_value["version"], yaml_value["url"], times.get_current_time(), pg_detail_id) + mysql.update(update_sql, val) + + +def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): + """ + Update origin data + """ + update_sql = "update cve_open_guss_yaml set origin_url = %s,status = %s, " \ + "cpe_name = %s, update_time = %s where id = %s" + val = (yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), pg_origin_id) + mysql.update(update_sql, val) + mysql.dispose() + + +def insert_yaml_origin_data(yaml_key, yaml_value, mysql): + """ + insert origin data + """ + insert_sql = "insert into cve_open_guss_yaml(package_name," \ + "version,origin_url, status, cpe_name,create_time,update_time) " \ + "values(%s,%s,%s,%s,%s,%s, %s)" + val = (yaml_key, yaml_value["version"], + yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), + times.get_current_time()) + last_id = mysql.insertOne(insert_sql, val) + mysql.dispose() + return last_id + + +def select_yaml_data(yaml_key, yaml_value, mysql): + """ + Query data + """ + sql = "SELECT package_id, git_id FROM cve_git_open_euler WHERE package_name = %s " \ + "and version = %s order by git_id desc" + val = (yaml_key, yaml_value["version"]) + packages_data = mysql.getMany(sql, val) + if packages_data and len(packages_data) > 0 and len(packages_data[0]) > 0: + return packages_data + else: + return None + + +def select_yaml_origin_data(yaml_key, yaml_value, mysql): + """ + Query origin data + """ + sql = "SELECT id FROM cve_open_guss_yaml WHERE package_name = %s " \ + "and version = %s" + val = (yaml_key, yaml_value["version"]) + packages_data = mysql.getOne(sql, val) + if packages_data and len(packages_data) > 0: + return packages_data + else: + return None + + +def select_yaml_lastdata(): + """ + Query the last data + """ + mysql = Mysql() + sql = "SELECT package_id FROM cve_git_open_euler order by git_id desc" + package_ids = mysql.getOne(sql) + if package_ids and len(package_ids) > 0: + return package_ids + else: + return None + + +def proc_gauss_yaml(): + """ + 1. Create a folder; + 2. Download yaml; + 3. Parse yaml; + 4. Save in mysql; + return None + """ + file_name = download_gauss_yaml() + if files.file_isexists(file_name): + yaml_data = parse_yaml(file_name) + if yaml_data != None and len(yaml_data) > 0: + store_yaml_data(yaml_data) + else: + print("File download failed, file: ", file_name) diff --git a/cve-py/tabletask/issue_statistics.py b/cve-py/tabletask/issue_statistics.py new file mode 100644 index 0000000..d9c6f08 --- /dev/null +++ b/cve-py/tabletask/issue_statistics.py @@ -0,0 +1,123 @@ +#!/usr/bin/python +# -*- coding: UTF-8 -*- + +################################################################################ +# +# Copyright (c) 2020 openEuler.org, Inc. All Rights Reserved +# +################################################################################ +""" + +Authors: zhangjianjun +Date: 01/18/2021 11:01 AM +""" + +import xlrd +import time +import os +import hashlib +import pymysql +from dbConnecttion.MysqlConn import Mysql +from downloadtask import downloadfiletask + + +def proc_excel_data(file_name): + """ + parse excel, import data + parse: file_name + return: list + """ + to_email_name_list = [] + cc_email_name_list = [] + email_dict = {} + try: + data = xlrd.open_workbook(file_name) + table_one = data.sheet_by_name("Sheet1") + row_number = table_one.nrows + for i in range(1, row_number): + if table_one.cell(i, 1).value == 1: + to_email_name_list.append(table_one.cell(i, 0).value) + else: + cc_email_name_list.append(table_one.cell(i, 0).value) + except IndexError as e: + print("Subscript out of bounds", e) + except xlrd.XLRDError as e: + print("Form not found：Sheet1", e) + email_dict["to"] = to_email_name_list + email_dict["cc"] = cc_email_name_list + return email_dict + + +def cur_date(): + """ + current date + :return createTime: string + """ + create_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + return create_time + + +def get_issue_excel(): + """ + import excel + :param + :return none + """ + file_dir = "issue_statistics" + downloadfiletask.download_excel(file_dir) + mysql = Mysql() + files = os.listdir('./' + file_dir) + for filename in files: + file_vaule = "./%s/%s" %(file_dir, filename) + with open(file_vaule, 'rb') as f: + sha1obj = hashlib.sha1() + sha1obj.update(f.read()) + hash_value = sha1obj.hexdigest() + print(filename, hash_value) + f.close() + sql = "select file_hash from cve_file_hash where file_name = %s" + val = (filename,) + file_hash = mysql.getOne(sql, val) + if file_hash: + if hash_value == file_hash['file_hash']: + print("File parsed:" + filename) + os.remove(file_vaule) + continue + sql = "insert into cve_file_hash (file_name, file_hash) values (%s, %s)" + val = (filename, hash_value) + mysql.insertOne(sql, val) + mysql.dispose() + email_dict = proc_excel_data(file_vaule) + if email_dict != None and len(email_dict) > 0: + to_email_name_list = email_dict["to"] + cc_email_name_list = email_dict["cc"] + else: + continue + if to_email_name_list != None and len(to_email_name_list) > 0: + del_sql = "delete from cve_issue_statistics_mail_list where email_type = %s" + mysql.delete(del_sql, (1, )) + mysql.dispose() + for email_name in to_email_name_list: + try: + insert_sql = "insert into cve_issue_statistics_mail_list(email_name, email_type, create_time) values(%s,%s, %s)" + val = (email_name, 1, cur_date()) + mysql.insertOne(insert_sql, val) + mysql.dispose() + except pymysql.err.IntegrityError as e: + print(e) + mysql.dispose(2) + del_sql = "delete from cve_issue_statistics_mail_list where email_type = %s" + mysql.delete(del_sql, (2, )) + mysql.dispose() + if cc_email_name_list != None and len(cc_email_name_list) > 0: + for email_name in cc_email_name_list: + try: + insert_sql = "insert into cve_issue_statistics_mail_list(email_name, email_type, create_time) values(%s,%s, %s)" + val = (email_name, 2, cur_date()) + mysql.insertOne(insert_sql, val) + mysql.dispose() + except pymysql.err.IntegrityError as e: + print(e) + mysql.dispose(2) + os.remove(file_vaule) + mysql.close() diff --git a/models/cve.go b/models/cve.go index 0e32d0b..46ee1e4 100644 --- a/models/cve.go +++ b/models/cve.go @@ -9,6 +9,10 @@ import ( "strings" ) +type SaNuberCount struct { + Count int64 +} + func QueryOriginCveExcel(days string, prcnum int) ([]OriginExcel, int64, error) { o := orm.NewOrm() var coe []OriginExcel @@ -716,6 +720,81 @@ func UnLockUpdateOriginExcelStatus(beforeStatus, afterStatus int, updateTime str } } +func GetSaNumCountByYears(years string, status int) int64 { + o := orm.NewOrm() + if status == 0 { + cnt, err := o.QueryTable("cve_sa_number").Filter("sa_years", years).Count() + //result, err := o.Raw("select count(sa_id) from cve_sa_number where sa_years = ?", years).Exec() + if err == nil { + logs.Info("cve_sa_number search result：", cnt) + } else { + logs.Error("cve_sa_number search result： failed") + } + return cnt + } else { + //err := o.Raw("select count(sa_id) from cve_sa_number where sa_years = ? and status = ?", years, status).QueryRow(&count) + cnt, err := o.QueryTable("cve_sa_number").Filter("sa_years", years).Filter("status", status).Count() + if err == nil { + logs.Info("cve_sa_number search result：", cnt) + } else { + logs.Error("cve_sa_number search result： failed") + } + return cnt + } +} + +func GetSaNumberByYears(sa *SaNumber, years string, status, orderSort int) error { + o := orm.NewOrm() + err := error(nil) + if status == 0 { + if orderSort == 1 { + err = o.Raw("select * from cve_sa_number where sa_years = ? order by sa_id asc limit 1", years).QueryRow(&sa) + logs.Info("GetSaNumberByYears", err) + } else { + err = o.Raw("select * from cve_sa_number where sa_years = ? order by sa_id desc limit 1", years).QueryRow(&sa) + logs.Info("GetSaNumberByYears", err) + } + } else { + if orderSort == 1 { + err = o.Raw("select * from cve_sa_number where sa_years = ? and status = ? order by sa_id asc limit 1", years, status).QueryRow(&sa) + logs.Info("GetSaNumberByYears", err) + } else { + err = o.Raw("select * from cve_sa_number where sa_years = ? and status = ? order by sa_id desc limit 1", years, status).QueryRow(&sa) + logs.Info("GetSaNumberByYears", err) + } + } + return err +} + +func DeleteSaNumberByYears(years string, status int) { + o := orm.NewOrm() + err := o.Raw("delete from cve_sa_number where sa_years != ? and status = ? ", years, status).QueryRow() + logs.Info("DeleteSaNumberByYears", err) +} + +func GetSaNumber(sa *SaNumber, field ...string) error { + o := orm.NewOrm() + err := o.Read(sa, field...) + return err +} + +func InsertSaNumber(sa *SaNumber) error { + o := orm.NewOrm() + id, err := o.Insert(sa) + logs.Info("InsertSaNumber, id: ", id, ", err: ", err) + return err +} + +func UpdateSaNumber(sa *SaNumber, field ...string) bool { + o := orm.NewOrm() + _, err := o.Update(sa, field...) + if err != nil { + logs.Error(err) + return false + } + return true +} + func UpdateVulnCenter(cve *VulnCenter, field ...string) bool { o := orm.NewOrm() _, err := o.Update(cve, field...) @@ -833,10 +912,10 @@ func GetCanExportExcelData(cveNum, issueNum string) (list []ExcelExport, err err sql := `SELECT b.num,c.*,a.issue_num,a.owned_component,a.cve_brief, d.sec_id,d.introduction,d.summary,d.theme,d.description,d.influence_component, d.affect_product,d.reference_link,d.affect_status, -e.public_date,openeuler_sa_num,a.cve_level +e.public_date,e.openeuler_sa_num,a.cve_level,b.organizate_id FROM cve_issue_template a RIGHT JOIN -(SELECT (SELECT COUNT(*) FROM cve_vuln_center WHERE cve_num = ?) num ,bc.cve_id,bc.cve_num +(SELECT (SELECT COUNT(*) FROM cve_vuln_center WHERE cve_num = ?) num ,bc.cve_id,bc.cve_num,bc.organizate_id FROM cve_vuln_center bc WHERE bc.cve_num = ? AND bc.is_export = 3 ) b ON a.cve_id = b.cve_id LEFT JOIN cve_score c diff --git a/models/excel.go b/models/excel.go index 9173342..f9f6f91 100644 --- a/models/excel.go +++ b/models/excel.go @@ -23,6 +23,7 @@ type ExcelExport struct { PublicDate string `json:"public_date"` OpenEulerSANum string `json:"openeuler_sa_num" orm:"size(128);column(openeuler_sa_num)"` CveLevel string `json:"cve_level" orm:"size(32);column(cve_level)"` + OrganizateId int8 `json:"organizate_id" orm:"column(organizate_id)"` } //ExcelPackage Released packages @@ -78,7 +79,7 @@ func GetCvrfRecord(cfr *CvrfSaRecord, colName ...string) error { return err } -func InsertCvrfRecord(cfr *CvrfSaRecord) (int64, error){ +func InsertCvrfRecord(cfr *CvrfSaRecord) (int64, error) { o := orm.NewOrm() num, err := o.Insert(cfr) return num, err @@ -88,4 +89,32 @@ func UpdateCvrfRecord(cfr *CvrfSaRecord, fields ...string) error { o := orm.NewOrm() _, err := o.Update(cfr, fields...) return err -} \ No newline at end of file +} + +func GetCvrfFileName(afl *SaFileList, colName ...string) error { + o := orm.NewOrm() + err := o.Read(afl, colName...) + return err +} + +func GetCvrfAllFile() []SaFileList { + o := orm.NewOrm() + var afl []SaFileList + num, err := o.Raw("SELECT file_name FROM cve_sa_file_list order by file_id asc").QueryRows(&afl) + if err == nil { + logs.Info("cve_sa_file_list nums: ", num) + } + return afl +} + +func InsertCvrfFileName(afl *SaFileList) (int64, error) { + o := orm.NewOrm() + num, err := o.Insert(afl) + return num, err +} + +func UpdateCvrfFileName(afl *SaFileList, fields ...string) error { + o := orm.NewOrm() + _, err := o.Update(afl, fields...) + return err +} diff --git a/models/issue.go b/models/issue.go index 337e392..94e25d3 100644 --- a/models/issue.go +++ b/models/issue.go @@ -58,7 +58,8 @@ func QueryIssueByPackName(packName, days string, prcnum int) ([]VulnCenter, erro o := orm.NewOrm() var vc []VulnCenter num, err := o.Raw("select cve_id,cve_num,cve_desc,cve_version,repair_time,"+ - "pack_name,cve_url,cve_level,data_source,update_time,is_export,cve_detail_url"+ + "pack_name,cve_url,cve_level,data_source,update_time,"+ + "is_export,cve_detail_url,organizate_id"+ " from cve_vuln_center where pack_name = ? and "+ "update_time >= ? and cve_status in (?, ?) "+ "order by cve_id asc limit ?", packName, days, 0, 1, prcnum).QueryRows(&vc) @@ -75,7 +76,7 @@ func QueryIssue(days string, prcnum int) ([]VulnCenter, error) { o := orm.NewOrm() var vc []VulnCenter num, err := o.Raw("select cve_id,cve_num,cve_desc,cve_version,repair_time,"+ - "pack_name,cve_url,cve_level,data_source,update_time,is_export,cve_detail_url"+ + "pack_name,cve_url,cve_level,data_source,update_time,is_export,cve_detail_url,organizate_id"+ " from cve_vuln_center where update_time >= ? and cve_status in (?, ?) "+ "order by cve_id asc limit ?", days, 0, 1, prcnum).QueryRows(&vc) if err == nil && num > 0 { @@ -174,7 +175,6 @@ func UpdateIssueTemplate(it *IssueTemplate, fields ...string) error { return err } - func UpdateScore(s *Score, fields ...string) error { o := orm.NewOrm() _, err := o.Update(s, fields...) @@ -522,3 +522,24 @@ func CreateIssueRecord(icr *IssueCreateRecord) (issueRecordId int64, err error) return icr.IssueRecordId, nil } } + +//QueryIssue query center +func QueryIssueTemplateByTime(beforeTime string, prcnum int, templateId int64, status int8) ([]IssueTemplate, error) { + o := orm.NewOrm() + var it []IssueTemplate + num, err := o.Raw("select *"+ + " from cve_issue_template where create_time >= ? and status < ? and template_id > ? "+ + "order by template_id asc limit ?", beforeTime, status, templateId, prcnum).QueryRows(&it) + if err == nil && num > 0 { + return it, nil + } + return it, err +} + +func QueryIssueStatisticEmail() ([]IssueStatisticsMailList, error) { + o := orm.NewOrm() + var ism []IssueStatisticsMailList + num, err := o.Raw("SELECT * FROM cve_issue_statistics_mail_list").QueryRows(&ism) + logs.Info("cve_issue_statistics_mail_list: ", num, err) + return ism, err +} diff --git a/models/modeldb.go b/models/modeldb.go index f155a45..36a8ce5 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -46,7 +46,7 @@ type VulnCenter struct { IsExport int8 `orm:"default(0);column(is_export)" description:"0:数据初始化;1:数据已同步到官网；2：数据不需要同步到官网；3：issue已关闭需要同步到官网"` DataSource int8 `orm:"default(1);column(data_source)" description:"1:软件所提供;2:网络爬取；3:人工处理; 4:兼容之前的数据"` CveDetailUrl string `orm:"size(1024);column(cve_detail_url)" description:"cve详情链接"` - OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:其他"` + OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:两者都有"` CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` DeleteTime time.Time `orm:"auto_now;type(datetime);column(delete_time)"` @@ -645,7 +645,7 @@ type CvrfSaRecord struct { Id int64 `orm:"pk;auto;column(id)"` OpenEulerSANum string `orm:"size(128);column(openeuler_sa_num);unique" description:"安全公告"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` - CveNum string `orm:"size(1024);column(cve_num);index" description:"cve编号"` + CveNum string `orm:"size(8192);column(cve_num);index" description:"cve编号"` XmlContent string `orm:"text;column(xml_content)" description:"xml内容"` Md5 string `orm:"size(128);column(cur_md5)" description:"当前内容生成的MD5"` UpdateMd5 string `orm:"size(128);column(update_md5)" description:"内容更新后生成的MD5"` @@ -657,6 +657,35 @@ type CvrfSaRecord struct { DeleteTime string `orm:"size(32);column(delete_time);null"` } +type SaNumber struct { + SaId int64 `orm:"pk;auto;column(sa_id)"` + OpenEulerSANum string `orm:"size(128);column(openeuler_sa_num);unique" description:"安全公告完整编号"` + SaYears string `orm:"size(16);column(sa_years);" description:"sa对应的年份"` + SaNum int64 `orm:"default(0);column(sa_num)" description:"sa编号"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已使用"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type SaFileList struct { + FileId int64 `orm:"pk;auto;column(file_id)"` + FileName string `orm:"size(512);column(file_name)" description:"sa的cvrf文件名称"` + Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + +type IssueStatisticsMailList struct { + Id int64 `orm:"pk;auto;column(id)"` + EmailName string `orm:"size(256);column(email_name);unique" description:"收件人"` + EmailType int8 `orm:"default(1);column(email_type)" description:"1:收件人;2:抄送人"` + CreateTime string `orm:"size(32);column(create_time)"` + UpdateTime string `orm:"size(32);column(update_time);null"` + DeleteTime string `orm:"size(32);column(delete_time);null"` +} + func CreateDb() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { @@ -686,6 +715,7 @@ func CreateDb() bool { new(PackageCpe), new(EmailList), new(IssueAssignee), new(IssueRepoWhitelist), new(IssueCreateRecord), new(OpenGussYaml), new(SpecIssueAssigness), new(SpecError), new(CvrfSaRecord), + new(SaNumber), new(SaFileList), new(IssueStatisticsMailList), ) logs.Info("table create success!") errosyn := orm.RunSyncdb("default", false, true) diff --git a/models/uploadcve.go b/models/uploadcve.go index 042e9ae..8e40e63 100644 --- a/models/uploadcve.go +++ b/models/uploadcve.go @@ -852,3 +852,10 @@ func QueryCveErrorInfo(issueStatus int8, startDate, endDate string) ([]IssueTemp return it, true } } + + +func GetOpengaussYaml(opy *OpenGussYaml, colName ...string) error { + o := orm.NewOrm() + err := o.Read(opy, colName...) + return err +} \ No newline at end of file diff --git a/task/hook.go b/task/hook.go index 9f009ce..cacccfb 100644 --- a/task/hook.go +++ b/task/hook.go @@ -3,6 +3,7 @@ package task import ( "cvevulner/common" "cvevulner/taskhandler" + "github.com/astaxie/beego" "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" "os" @@ -20,9 +21,11 @@ func ProcHookEvent() error { hookurl := BConfig.String("hook::hookurl") owner := BConfig.String("gitee::owner") accessToken := os.Getenv("GITEE_TOKEN") + gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") + gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") pwd := BConfig.String("hook::hookpwd") // Get the data source of the table - err = taskhandler.ProcHookEvent(hookurl, owner, accessToken, pwd) + err = taskhandler.ProcHookEvent(hookurl, owner, accessToken, pwd, gaussOwner, gitGaussToken) logs.Info("处理hook事件 task end") return err } diff --git a/task/inittask.go b/task/inittask.go index 9077f9a..771906e 100644 --- a/task/inittask.go +++ b/task/inittask.go @@ -86,6 +86,22 @@ func AbnormalCveTask(exceptcve string) { logs.Info("创建处理异常cve任务 task end") } +// Generate SA number +func GenerateSaNumberTask(genSa string) { + logs.Info("生成SA编号 task start") + proSaTask := toolbox.NewTask("GenerateSaNumber", genSa, ProcSaNumber) + toolbox.AddTask("GenerateSaNumber", proSaTask) + logs.Info("生成SA编号 task end") +} + +// issue statistics mailing list +func IssueStstisticsTask(issueStatistic string) { + logs.Info("issue统计 task start") + iStTask := toolbox.NewTask("IssueStatistics", issueStatistic, IssueStatistics) + toolbox.AddTask("IssueStatistics", iStTask) + logs.Info("issue统计 task end") +} + // Print logs to the console and delete redundant logs func PrintLogTask(printLog string) { logs.Info("创建日志任务 task start") @@ -190,5 +206,21 @@ func InitTask() bool { } else { logs.Info("exceptcve no task") } + // Generate SA number + gensaflag, errx := BConfig.Int("crontab::gensaflag") + if gensaflag == 1 && errx == nil { + gensa := BConfig.String("crontab::gensa") + GenerateSaNumberTask(gensa) + } else { + logs.Info("Generate SA task") + } + // issue statistics mailing list + issuestatisticflag, errx := BConfig.Int("crontab::issuestatisticflag") + if issuestatisticflag == 1 && errx == nil { + issueStatistic := BConfig.String("crontab::issuestatistic") + IssueStstisticsTask(issueStatistic) + } else { + logs.Info("issue statistics task") + } return true } diff --git a/task/issuestatistics.go b/task/issuestatistics.go new file mode 100644 index 0000000..8cc0467 --- /dev/null +++ b/task/issuestatistics.go @@ -0,0 +1,29 @@ +package task + +import ( + "cvevulner/common" + "cvevulner/taskhandler" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" +) + +// issue statistics mailing list +func IssueStatistics() error { + defer common.Catchs() + logs.Info("issue statistics task start") + // The number of days the issue counts the mailing list + beforeDate, beErr := beego.AppConfig.Int("cve::cve_statistics_date") + if beErr != nil { + beforeDate = 30 + } + prnum, prErr := beego.AppConfig.Int("crontab::prcnum") + if prErr != nil { + prnum = 100 + } + owner := beego.AppConfig.String("gitee::owner") + // Get the data source of the table + err := taskhandler.IssueStatistics(beforeDate, prnum, owner) + logs.Info(err) + logs.Info("issue statistics task end") + return err +} diff --git a/task/issuetask.go b/task/issuetask.go index a0d7b13..1a1f9d7 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -86,7 +86,8 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears return errors.New("Failed to get token in environment variable") } owner := beego.AppConfig.String("gitee::owner") - path := beego.AppConfig.String("gitee::path") + gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") + gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") for ; ; { cveData, err := models.QueryIssue(beforeTime, prcnum) if err == nil && len(cveData) > 0 { @@ -142,12 +143,14 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears logs.Error("Failed to get Score, err: ", err, "cveId: ", issueValue.CveId) continue } - GenIssueRecordData(&icr, issueValue, sc, 2) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) + if issueValue.OrganizationID != 2 { + GenIssueRecordData(&icr, issueValue, sc, 2) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } } continue } @@ -165,7 +168,7 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears if issueValue.Status == 0 { issueValue.Status = 2 mutex.Lock() - err := ProcIssue(issueValue, accessToken, owner, path) + err := ProcIssue(issueValue, accessToken, owner, gaussOwner, gitGaussToken) mutex.Unlock() if err != nil { logs.Error("Failed to create issue, cvenum: ", issueValue.CveNum, "err,err: ", err) @@ -174,7 +177,7 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears } else { issueValue.Status = 2 mutex.Lock() - err := ProcUpdateIssue(issueValue, accessToken, owner, path) + err := ProcUpdateIssue(issueValue, accessToken, owner) mutex.Unlock() if err != nil { logs.Error("Failed to update issue, cvenum: ", issueValue.CveNum, "err,err: ", err) @@ -188,12 +191,14 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears logs.Error("Failed to get Score, err: ", err, "cveId: ", issueValue.CveId) continue } - GenIssueRecordData(&icr, issueValue, sc, 1) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) + if issueValue.OrganizationID != 2 { + GenIssueRecordData(&icr, issueValue, sc, 1) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } } } } @@ -206,7 +211,8 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i return errors.New("Failed to get token in environment variable") } owner := beego.AppConfig.String("gitee::owner") - path := beego.AppConfig.String("gitee::path") + gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") + gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") issueWhiteData, issueErr := models.QueryIssueWhitelist(0) if issueErr == nil && len(issueWhiteData) > 0 { for _, issues := range issueWhiteData { @@ -229,14 +235,17 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i // add mutex lockErr := models.LockUpdateIssueStatus(issueValue.CveId, issueValue.CveNum, 15) if !lockErr { - logs.Error("The current cve is processing, continue to process the next data, err: ", lockErr, ",data: ", issueValue) + logs.Error("The current cve is processing, continue to"+ + " process the next data, err: ", lockErr, ",data: ", issueValue) continue } - se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, PackName: issueValue.PackName, Status: 1} + se := models.SpecError{CveNum: issueValue.CveNum, Owner: owner, + PackName: issueValue.PackName, Status: 1} seError := models.GetIssueSpecErrByColName(&se, "CveNum", "Owner", "PackName", "Status") if seError == nil && se.Id > 0 { models.UpdateIssueStatus(issueValue, 5) - logs.Info("The current issue does not need to be processed, it has been processed, cveData: ", issueValue) + logs.Info("The current issue does not need to be processed,"+ + " it has been processed, cveData: ", issueValue) continue } // Determine whether the issue has been processed @@ -269,12 +278,14 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) continue } - GenIssueRecordData(&icr, issueValue, sc, 2) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) + if issueValue.OrganizationID != 2 { + GenIssueRecordData(&icr, issueValue, sc, 2) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } } continue } @@ -290,7 +301,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i // Process each piece of cve data if issueValue.Status == 0 { mutex.Lock() - err := ProcIssue(issueValue, accessToken, owner, path) + err := ProcIssue(issueValue, accessToken, owner, gaussOwner, gitGaussToken) mutex.Unlock() if err != nil { logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) @@ -298,7 +309,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i } } else { mutex.Lock() - err := ProcUpdateIssue(issueValue, accessToken, owner, path) + err := ProcUpdateIssue(issueValue, accessToken, owner) mutex.Unlock() if err != nil { logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) @@ -312,12 +323,14 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) continue } - GenIssueRecordData(&icr, issueValue, sc, 1) - issueRecordId, issReErr := models.CreateIssueRecord(&icr) - if issReErr == nil && issueRecordId > 0 { - logs.Info("Issue record data created successfully, id:", issueRecordId) - } else { - logs.Error("Failed to create issue record data, err: ", issReErr) + if issueValue.OrganizationID != 2 { + GenIssueRecordData(&icr, issueValue, sc, 1) + issueRecordId, issReErr := models.CreateIssueRecord(&icr) + if issReErr == nil && issueRecordId > 0 { + logs.Info("Issue record data created successfully, id:", issueRecordId) + } else { + logs.Error("Failed to create issue record data, err: ", issReErr) + } } } } @@ -384,11 +397,12 @@ func CreateIssue() error { } //ProcUpdateIssue Update issue -func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path string) error { +func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) error { + path := issueValue.PackName // Query and modify score sr, err := models.QueryIssueScoreRecord(issueValue.CveId, 0) if err != nil { - logs.Error("查询 评分记录失败, cveId: ", issueValue.CveId, "err: ", err) + logs.Error("Failed to query score records, cveId: ", issueValue.CveId, "err: ", err) return err } // Query issue template @@ -403,7 +417,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri it.NVDVector = sr.NvectorVule it.CveBrief = issueValue.Description it.CveLevel = issueValue.CveLevel - if it.Assignee == "" || len(it.Assignee) < 2 { + if (it.Assignee == "" || len(it.Assignee) < 2) && issueValue.OrganizationID != 2 { repoMainTainer, ok := models.QueryRepoMaintainer(issueValue.PackName) if !ok || repoMainTainer == "" || len(repoMainTainer) < 1 { assignee, assErr := taskhandler.GetCollaboratorInfo(accessToken, owner, path) @@ -421,14 +435,13 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri // Judgment of necessary fields if it.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { - logs.Error("Field is empty: NVDScore: ", it.NVDScore,",Description: ", + logs.Error("Field is empty: NVDScore: ", it.NVDScore, ",Description: ", issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) // Update issue status models.UpdateIssueStatus(issueValue, 6) return errors.New("Field is empty") } - path = it.Repo _, err := taskhandler.UpdateIssueToGit(accessToken, owner, path, issueValue, it) if err != nil && err.Error() != "Recreate issue" { @@ -453,63 +466,89 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri } //ProcIssue Deal with issue -func ProcIssue(issueValue models.VulnCenter, accessToken, owner, path string) error { +func ProcIssue(issueValue models.VulnCenter, + accessToken, owner, gaussOwner, gitGaussToken string) error { assignee := "" + path := issueValue.PackName brandArry := []string{} - sn, err := models.QueryIssueSecurity(issueValue.CveId, issueValue.CveNum) - if err == nil && len(sn.CveNum) > 0 { - path = issueValue.PackName - logs.Info("查询安全信息：sn: ", sn) - // Get issue handler - repoMainTainer, ok := models.QueryRepoMaintainer(issueValue.PackName) - if !ok || repoMainTainer == "" || len(repoMainTainer) < 1 { - assignee, err = taskhandler.GetCollaboratorInfo(accessToken, owner, path) - if assignee == "" { - logs.Error("获取仓库： owner:", owner, "path:", path, "分析人失败", "err:", err, - "cveid: ", issueValue.CveId, ",创建无maintainer的issue") - //return err + if issueValue.OrganizationID != 2 { + sn, err := models.QueryIssueSecurity(issueValue.CveId, issueValue.CveNum) + if err == nil && len(sn.CveNum) > 0 { + logs.Info("查询安全信息：sn: ", sn) + // Get issue handler + repoMainTainer, ok := models.QueryRepoMaintainer(issueValue.PackName) + if !ok || repoMainTainer == "" || len(repoMainTainer) < 1 { + assignee, err = taskhandler.GetCollaboratorInfo(accessToken, owner, path) + if assignee == "" { + logs.Error("获取仓库： owner:", owner, "path:", path, "分析人失败", "err:", err, + "cveid: ", issueValue.CveId, ",创建无maintainer的issue") + } + } else { + assignee = repoMainTainer } } else { - assignee = repoMainTainer + logs.Error("获取security 失败, err: ", err, "cveId: ", issueValue.CveId) + return err + } + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + return err + } + branchList := []string{} + errBrands := errors.New("") + // Get branch information + branchList, errBrands = taskhandler.GetBranchesInfo(accessToken, owner, path) + if branchList == nil || len(branchList) == 0 { + logs.Error("获取分支信息失败,CveNum: ", issueValue.CveNum, "path: ", path, ", err: ", errBrands) + } + if len(branchList) > 0 { + brandArry = []string{} + for _, branch := range branchList { + brandArry = append(brandArry, branch) + } + } + // Judgment of necessary fields + if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + logs.Error("Field is empty: NVDScore: ", sc.NVDScore, ",Description: ", + issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", + issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) + // Update issue status + models.UpdateIssueStatus(issueValue, 6) + return errors.New("Field is empty") + } + resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, brandArry) + if err != nil && err.Error() != "Recreate issue" { + logs.Error("创建issue失败, err: ", err, "resp: ", resp, "cveId: ", issueValue.CveId) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) + return err } } else { - logs.Error("获取security 失败, err: ", err, "cveId: ", issueValue.CveId) - return err - } - sc, err := models.QueryIssueScore(issueValue.CveId) - if err != nil { - logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) - return err - } - branchList := []string{} - errBrands := errors.New("") - // Get branch information - branchList, errBrands = taskhandler.GetBranchesInfo(accessToken, owner, path) - if branchList == nil || len(branchList) == 0 { - logs.Error("获取分支信息失败,CveNum: ", issueValue.CveNum, "path: ", path, ", err: ", errBrands) - } - if len(branchList) > 0 { - brandArry = []string{} - for _, branch := range branchList { - brandArry = append(brandArry, branch) + sc, err := models.QueryIssueScore(issueValue.CveId) + if err != nil { + logs.Error("获取Score 失败, err: ", err, "cveId: ", issueValue.CveId) + return err + } + // Judgment of necessary fields + if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || + len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { + logs.Error("Field is empty: NVDScore: ", sc.NVDScore, ",Description: ", + issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", + issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) + // Update issue status + models.UpdateIssueStatus(issueValue, 6) + return errors.New("Field is empty") + } + path = "security" + resp, err := taskhandler.CreateIssueToGit(gitGaussToken, gaussOwner, path, assignee, issueValue, sc, brandArry) + if err != nil && err.Error() != "Recreate issue" { + logs.Error("创建issue失败, err: ", err, "resp: ", resp, "cveId: ", issueValue.CveId) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) + return err } - } - // Judgment of necessary fields - if sc.NVDScore <= 0 || len(issueValue.Description) < 2 || len(issueValue.CveNum) < 2 || - len(issueValue.PackName) < 2 || len(issueValue.CveVersion) < 1 { - logs.Error("Field is empty: NVDScore: ", sc.NVDScore,",Description: ", - issueValue.Description, ",CveNum:", issueValue.CveNum, ",PackName: ", - issueValue.PackName, ",CveVersion: ", issueValue.CveVersion) - // Update issue status - models.UpdateIssueStatus(issueValue, 6) - return errors.New("Field is empty") - } - resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, brandArry) - if err != nil && err.Error() != "Recreate issue" { - logs.Error("创建issue失败, err: ", err, "resp: ", resp, "cveId: ", issueValue.CveId) - // Update issue status - models.UpdateIssueStatus(issueValue, 3) - return err } return nil } diff --git a/task/oricvecheck.go b/task/oricvecheck.go index 3765a09..8d236ed 100644 --- a/task/oricvecheck.go +++ b/task/oricvecheck.go @@ -48,7 +48,6 @@ func UnLockTable() error { return nil } - // Cve handles abnormal data reprocessing func ProcAbnCve() error { defer common.Catchs() @@ -85,3 +84,25 @@ func ProcAbnCve() error { logs.Info("纠正cve数据 task end") return err } + +// Generate a certain number of SA numbers each time +func ProcSaNumber() error { + defer common.Catchs() + logs.Info("Generate SA number task start") + // Query the cve to be processed, 1: add; 2: modify + BConfig, err := config.NewConfig("ini", "conf/app.conf") + if err != nil { + logs.Error("config init error:", err) + return err + } + // The amount of data processed at a time + saReAmount, err := BConfig.Int64("cve::sa_re_amount") + if err != nil { + logs.Error("config cve::sa_re_amount error: invalid value is ", saReAmount) + return errors.New("value is nil") + } + // Get the data source of the table + _, err = taskhandler.AddSaNumber(saReAmount) + logs.Info("Generate SA number task end") + return err +} diff --git a/taskhandler/assist.go b/taskhandler/assist.go index 92bc7a5..2be2cbc 100644 --- a/taskhandler/assist.go +++ b/taskhandler/assist.go @@ -6,7 +6,6 @@ import ( "errors" "github.com/astaxie/beego/logs" "os" - "strings" ) func GetOautToken(gt GiteeToken) { @@ -89,8 +88,9 @@ func GetBranchesInfo(accessToken string, owner string, path string) ([]string, e } mapValue := value["name"].(string) if mapValue != "" && len(mapValue) > 3 { - subStr := mapValue[len(mapValue)-3:] - if strings.ToUpper(subStr) == "LTS" || mapValue == "openEuler-20.09"{ + if mapValue == "openEuler-20.03-LTS" || + mapValue == "openEuler-20.09" || + mapValue == "openEuler-20.03-LTS-SP1"{ branchNameList = append(branchNameList, mapValue) } } diff --git a/taskhandler/common.go b/taskhandler/common.go index 451e81f..88f526e 100644 --- a/taskhandler/common.go +++ b/taskhandler/common.go @@ -101,6 +101,77 @@ const commentCopyValue = ` openEuler评分: (评分和向量) +受影响版本排查(受影响/不受影响): +%v +----------------------------------------------------------------------- +issue处理具体操作请参考: +%v +pr关联issue具体操作请参考: +%v +` + +const gaussBodyTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞信息修改时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openGauss评分： + %v + 受影响版本排查(受影响/不受影响)： + %v +` + +const gaussBodyUpTpl = `一、漏洞信息 + 漏洞编号：%v + 漏洞归属组件：%v + 漏洞归属的版本：%v + CVSS V%v分值： + BaseScore：%v + Vector：CVSS：%v + 漏洞简述： + %v + 漏洞公开时间：%v + 漏洞信息修改时间：%v + 漏洞详情参考链接： + %v + 漏洞分析指导链接： + %v +二、漏洞分析结构反馈 + 影响性分析说明： + %v + openGauss评分： + %v + Vector：CVSS：%v + 受影响版本排查(受影响/不受影响)： + %v +` + +const gaussCommentCopyValue = ` +%v +**issue处理注意事项:** +**1. 当前issue受影响的分支提交pr时, 须在pr描述中填写当前issue编号进行关联, 否则无法关闭当前issue;** +**2. 模板内容需要填写完整, 无论是受影响或者不受影响都需要填写完整内容, 否则无法关闭当前issue;** +**3. 以下为模板中需要填写完整的内容, 请复制到评论区回复, 注: 内容的标题名称(影响性分析说明, openEuler评分, 受影响版本排查(受影响/不受影响))不能省略,省略后cve-manager将无法正常解析填写内容.** +************************************************************************ +影响性分析说明: + + +openGauss评分: (评分和向量) + + 受影响版本排查(受影响/不受影响): %v ----------------------------------------------------------------------- @@ -130,6 +201,12 @@ func CommentTemplate(assignee, commentCmd, affectedVersion, path string) string return commentTemplate } +func GaussCommentTemplate(assignee, commentCmd, affectedVersion string) string { + assigneeStr := "@" + assignee + commentTemplate := fmt.Sprintf(gaussCommentCopyValue, assigneeStr, affectedVersion, commentCmd, PrIssueLink) + return commentTemplate +} + func GitOpenEulerData(values map[string]interface{}, ge *models.GitOpenEuler, tb models.GitPackageTable) { defer common.Catchs() ge.TableName = tb.TableName @@ -399,12 +476,15 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str } } updateTime := cve.UpdateTime.String() + updateTime = common.GetLocalCurTime() + logs.Info("cve.UpdateTime.String(): ", cve.UpdateTime.String(), cve.UpdateTime) if updateTime != "" && len(updateTime) > 19 { updateTime = updateTime[:19] } StatusName := "" if its.StatusName != "" && len(its.StatusName) > 1 { - if its.StatusName == "待办的" || its.StatusName == "开启的" || strings.ToLower(its.StatusName) == "open" { + if its.StatusName == "待办的" || its.StatusName == "开启的" || + strings.ToLower(its.StatusName) == "open" { StatusName = "open" } else if its.StatusName == "进行中" || strings.ToLower(its.StatusName) == "started" || strings.ToLower(its.StatusName) == "progressing" { @@ -423,13 +503,19 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str return "" } } + bodyUpTplx := bodyUpTpl + bodyTplx := bodyTpl + if cve.OrganizationID == 2 { + bodyUpTplx = gaussBodyUpTpl + bodyTplx = gaussBodyTpl + } if flag == 1 { if floatOpenEulerScore > 0.0 || (its.OpenEulerVector != "" && len(its.OpenEulerVector) > 1) { cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" nveScore := score + " " + cve.CveLevel nveVector := scoreType + "/" + sc.NvectorVule oVector := scoreType + "/" + its.OpenEulerVector - body := fmt.Sprintf(bodyUpTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, + body := fmt.Sprintf(bodyUpTplx, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, oVector, affectedVersion) requestBody = fmt.Sprintf(`{ @@ -447,7 +533,7 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str nveScore := score + " " + cve.CveLevel nveVector := scoreType + "/" + sc.NvectorVule openEulerScore = " " - body := fmt.Sprintf(bodyTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, + body := fmt.Sprintf(bodyTplx, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, affectedVersion) requestBody = fmt.Sprintf(`{ @@ -466,7 +552,7 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str nveScore := score + " " + cve.CveLevel nveVector := scoreType + "/" + sc.NvectorVule openEulerScore = " " - body := fmt.Sprintf(bodyTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, + body := fmt.Sprintf(bodyTplx, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, affectedVersion) requestBody = fmt.Sprintf(`{ @@ -485,7 +571,7 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str nveScore := score + " " + cve.CveLevel nveVector := scoreType + "/" + its.NVDVector oVector := scoreType + "/" + its.OpenEulerVector - body := fmt.Sprintf(bodyUpTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, + body := fmt.Sprintf(bodyUpTplx, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, oVector, affectedVersion) requestBody = fmt.Sprintf(`{ @@ -503,7 +589,7 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str cveNumber := "[" + cve.CveNum + "](https://nvd.nist.gov/vuln/detail/" + cve.CveNum + ")" nveScore := score + " " + cve.CveLevel nveVector := scoreType + "/" + sc.NvectorVule - body := fmt.Sprintf(bodyTpl, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, + body := fmt.Sprintf(bodyTplx, cveNumber, path, cve.CveVersion, scoreType, nveScore, nveVector, cve.Description, cve.RepairTime, updateTime, cve.CveDetailUrl, commentCmd, cveAnalysis, openEulerScore, affectedVersion) requestBody = fmt.Sprintf(`{ @@ -552,7 +638,7 @@ func RemoveSubstring(s string, subList []string) string { func deleteTailBlank(str string) string { spaceNum := 0 - for i := len(str) - 1; i >= 0; i-- { // 去除字符串尾部的所有空格 + for i := len(str) - 1; i >= 0; i-- { if str[i] == ' ' { spaceNum++ } else { @@ -575,6 +661,17 @@ func openEulerScoreProc(openEulerScore float64) (CveLevel string) { return CveLevel } +func IssueStateConversion(status int8) string { + var retName string + switch status { + case 2: + retName = "进行中" + default: + retName = "待办的" + } + return retName +} + func PathExists(path string) (bool, error) { _, err := os.Stat(path) if err == nil { diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index 13f4509..8ccea0b 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -317,6 +317,13 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, func UpdateIssueToGit(accessToken string, owner string, path string, cve models.VulnCenter, its models.IssueTemplate) (string, error) { logs.Info("更新模板请求参数: cve: ", cve, ",its: ", its, ", owner: ", owner, ",path: ", path) + gaussOwner := beego.AppConfig.String("opengauss::gauss_owner") + gitGaussToken := beego.AppConfig.String("opengauss::git_gauss_token") + if cve.OrganizationID == 2 { + owner = gaussOwner + accessToken = gitGaussToken + path = "security" + } if its.IssueNum != "" && len(its.IssueNum) > 0 { issueErr := GetIssueNum(accessToken, owner, path, its.IssueNum) if issueErr != nil { @@ -588,6 +595,9 @@ func CreateIssueComment(accessToken, owner, path, assignee string, } commentCmd := BConfig.String("reflink::comment_cmd") commentBody := CommentTemplate(assignee, commentCmd, affectedVersion, path) + if cve.OrganizationID == 2 { + commentBody = GaussCommentTemplate(assignee, commentCmd, affectedVersion) + } requestBody := fmt.Sprintf(`{ "access_token": "%s", "body": "%s" diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 182f1d3..1eb9289 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -31,7 +31,7 @@ var ( var ewg sync.WaitGroup func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNum int, CveRes models.VulnCenter, - goe models.GitPackageInfo) (bool, error) { + goe models.GitPackageInfo, organizationID int8) (bool, error) { var OpenEulId int64 CveRes.Description = cveData.CveDesc CveRes.CveVersion = cveData.CveVersion @@ -39,6 +39,7 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNu CveRes.PackName = cveData.PackName CveRes.CveUrl = cveRef + cveData.CveNum CveRes.CveLevel = cveData.CveLevel + CveRes.OrganizationID = organizationID //CveRes.DataSource = 3 if CveRes.DataSource != 4 && CveRes.Status != 0 && CveRes.Status != 1 { CveRes.Status = 1 @@ -207,7 +208,8 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNu return true, nil } -func InsertCveExcelGroups(cveData models.OriginExcel, cveRef string, openeulerNum int, goe models.GitPackageInfo) (bool, error) { +func InsertCveExcelGroups(cveData models.OriginExcel, cveRef string, openeulerNum int, + goe models.GitPackageInfo, organizationID int8) (bool, error) { var vul models.VulnCenter vul.CveNum = cveData.CveNum vul.Description = cveData.CveDesc @@ -219,6 +221,7 @@ func InsertCveExcelGroups(cveData models.OriginExcel, cveRef string, openeulerNu vul.CveLevel = cveData.CveLevel vul.DataSource = 3 vul.IsExport = 0 + vul.OrganizationID = organizationID var sec models.SecurityNotice sec.CveNum = cveData.CveNum sec.InfluenceComponent = cveData.PackName @@ -306,7 +309,7 @@ func InsertCveExcelGroups(cveData models.OriginExcel, cveRef string, openeulerNu func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum int, CveRes models.VulnCenter, cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, - scopeType string, cveScV2 models.OriginUpstreamImpactScoreV2, pkList []string) (bool, error) { + scopeType string, cveScV2 models.OriginUpstreamImpactScoreV2, pkList []string, organizationID int8) (bool, error) { var OpenEulId int64 CveRes.Description = cveDesc.EnDescription CveRes.CveVersion = pkList[1] @@ -314,6 +317,7 @@ func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum CveRes.PackName = pkList[0] CveRes.CveUrl = cveRef + cveData.CveNum CveRes.CveLevel = cveScV3.CveLevel + CveRes.OrganizationID = organizationID //CveRes.DataSource = 1 if CveRes.DataSource != 4 && CveRes.Status != 0 && CveRes.Status != 1 { CveRes.Status = 1 @@ -548,7 +552,7 @@ func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum func InsertCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum int, cveDesc models.OriginUpstreamDesc, cveScV3 models.OriginUpstreamImpactScoreV3, goe models.GitPackageInfo, - scopeType string, cveScV2 models.OriginUpstreamImpactScoreV2, pkList []string) (bool, error) { + scopeType string, cveScV2 models.OriginUpstreamImpactScoreV2, pkList []string, organizationID int8) (bool, error) { var vul models.VulnCenter vul.CveNum = cveData.CveNum vul.Description = cveDesc.EnDescription @@ -560,6 +564,7 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum vul.CveLevel = cveScV3.CveLevel vul.DataSource = 1 vul.IsExport = 0 + vul.OrganizationID = organizationID var sec models.SecurityNotice sec.CveNum = cveData.CveNum sec.InfluenceComponent = pkList[0] @@ -767,12 +772,18 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) pkList = append(pkList, key) pkList = append(pkList, value) versionFlag := false + gaussFlag := false var goe models.GitPackageInfo if value != "" && len(value) > 0 { versionList := strings.Split(value, ",") if len(versionList) > 0 { for _, ver := range versionList { if ver != "" && len(ver) > 1 { + opy := models.OpenGussYaml{PackageName: key, Version: ver} + openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + if openErr == nil && opy.Id > 0 { + gaussFlag = true + } goe, ok = models.QueryCveOpeneulerDetaildataByName(key, ver) if ok { versionFlag = true @@ -790,14 +801,23 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) repoFlag = true } if !versionFlag || !repoFlag { - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 4) - logs.Error("版本信息对应错误, ", cveData.PackName, cveData.Version, cveData.CveId) - return false, errors.New("数据错误，暂时不处理") + if !gaussFlag { + models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 4) + logs.Error("版本信息对应错误, ", cveData.PackName, cveData.Version, cveData.CveId) + return false, errors.New("数据错误，暂时不处理") + } + } + organizationID := int8(1) + if versionFlag && gaussFlag { + organizationID = 3 + } else if !versionFlag && gaussFlag { + organizationID = 2 } CveRes, err := models.QueryCveByNum(cveData.CveNum, key, value) if err { lockx.Lock() - ok, err := UpdateCveGroups(cveData, cveRef, openeulernum, CveRes, cveDesc, cveScV3, goe, scopeType, cveScV2, pkList) + ok, err := UpdateCveGroups(cveData, cveRef, openeulernum, CveRes, cveDesc, cveScV3, goe, + scopeType, cveScV2, pkList, organizationID) lockx.Unlock() if !ok { logs.Error("更新cve数据失败, cveData: ", cveData, ", err: ", err) @@ -806,7 +826,8 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) } } else { lockx.Lock() - ok, err := InsertCveGroups(cveData, cveRef, openeulernum, cveDesc, cveScV3, goe, scopeType, cveScV2, pkList) + ok, err := InsertCveGroups(cveData, cveRef, openeulernum, cveDesc, cveScV3, goe, + scopeType, cveScV2, pkList, organizationID) lockx.Unlock() if !ok { logs.Error("插入cve数据失败, cveData: ", cveData, ", err: ", err) @@ -826,11 +847,23 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 3) return false, errors.New("数据错误，暂时不处理") } + organizationID := int8(1) + opy := models.OpenGussYaml{PackageName: cveData.PackName, Version: cveData.CveVersion} + openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + if openErr == nil && opy.Id > 0 { + organizationID = 2 + } god, ok := models.QueryCveOpeneulerdata(cveData.PackName, cveData.CveVersion) if !ok { - logs.Error("The repo and version correspond to errors, data: ", cveData) - models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 4) - return false, errors.New("The repo and version correspond to errors and will not be processed temporarily") + if organizationID == 1 { + logs.Error("The repo and version correspond to errors, data: ", cveData) + models.UpdateOriginExcelStatus(common.GetCurTime(), cveData.CveId, 4) + return false, errors.New("The repo and version correspond to errors and will not be processed temporarily") + } + } else { + if organizationID == 2 { + organizationID = 3 + } } // Import cve as data after 2018 cveNumList := strings.Split(cveData.CveNum, "-") @@ -851,7 +884,7 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe CveRes, err := models.QueryCveByNum(cveData.CveNum, cveData.PackName, cveData.CveVersion) if err { lockx.Lock() - ok, err := UpdateExcelCveGroups(cveData, cveRef, openeulerNum, CveRes, goe) + ok, err := UpdateExcelCveGroups(cveData, cveRef, openeulerNum, CveRes, goe, organizationID) lockx.Unlock() if !ok { logs.Error("更新cve数据失败, cveData: ", cveData, ", err: ", err) @@ -860,7 +893,7 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe } } else { lockx.Lock() - ok, err := InsertCveExcelGroups(cveData, cveRef, openeulerNum, goe) + ok, err := InsertCveExcelGroups(cveData, cveRef, openeulerNum, goe, organizationID) lockx.Unlock() if !ok { logs.Error("插入cve数据失败, cveData: ", cveData, ", err: ", err) @@ -889,9 +922,11 @@ func GetCveOriginData(prcnum, days, openeulernum int, cveRef string) (bool, erro count = count + 1 logs.Info("当前正常解析第： ", count, "条数据,i:", i, ", cvenum: ", cveOrg.Ids) // Add mutex - lockOk := models.LockUpdateOriginStatus(common.GetCurTime(), cveOrg.PackName, cveOrg.Version, cveOrg.CveId, 15) + lockOk := models.LockUpdateOriginStatus(common.GetCurTime(), cveOrg.PackName, + cveOrg.Version, cveOrg.CveId, 15) if !lockOk { - logs.Error("Current data is being processed: PackName: ", cveOrg.PackName, cveOrg.Version, cveOrg.CveId) + logs.Error("Current data is being processed: PackName: ", + cveOrg.PackName, cveOrg.Version, cveOrg.CveId) ch <- i continue } @@ -969,6 +1004,7 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c vul.CveUrl = cveRef + cveData.CveNumber vul.IsExport = 0 vul.DataSource = 4 + vul.OrganizationID = 1 v2, cvsError := strconv.ParseFloat(lop.CvsScore, 64) if cvsError != nil { vul.CveLevel = "Critical" @@ -1918,7 +1954,7 @@ func GenerateExcelTrigger(wgx *sync.WaitGroup, fileName, startTime, fileCode, af logs.Error("File compression failed: err: ", zipErr) } // send email - sendError := SendEmail(zipFileName, 2) + sendError := SendEmail(zipFileName, 0, "") if sendError != nil { logs.Error("SendEmail, sendErr: ", sendError) return diff --git a/taskhandler/cvrf.go b/taskhandler/cvrf.go index eb033a0..2456326 100644 --- a/taskhandler/cvrf.go +++ b/taskhandler/cvrf.go @@ -9,7 +9,10 @@ import ( "errors" "fmt" "github.com/astaxie/beego/logs" + "io" "io/ioutil" + "mime/multipart" + "net/http" "os" "sort" "strconv" @@ -198,8 +201,8 @@ type Remediations struct { } type UnRemediations struct { - XMLName xml.Name `xml:"Remediations,omitempty"` - Remediation *UnRemediation `xml:"Remediation,omitempty"` + XMLName xml.Name `xml:"Remediations,omitempty"` + Remediation []UnRemediation `xml:"Remediation,omitempty"` } type Remediation struct { @@ -215,6 +218,7 @@ type UnRemediation struct { Type string `xml:"Type,attr"` Description string `xml:"Description"` Date string `xml:"DATE"` + ProductId string `xml:"ProductID"` } type CveInfo struct { @@ -255,13 +259,20 @@ type CveCvrf struct { CveInfo CveInfo } +type BrachCveInfo struct { + OpenEulerSANum string + CvrfFileName string + CveNumSlice []string +} + type ComponentInfo struct { OpenEulerSANum string OwnedComponent string CveNum []string - CveNumMap map[string][]string + CveNumMap map[string]BrachCveInfo OpenEulerScore []float64 UpdateFlag int + CvrfFileName string } type UnaffectCvrfSa struct { @@ -387,13 +398,15 @@ func BuildUnaffectVulnerabilitySet(unaffectCvrfsa *UnaffectCvrfSa, v models.Exce func BuildUnaffVulnerabilitySlice(vulnerability []UnaffectVulnerability, v models.ExcelExport, affectBranch string, componentMap map[string]ComponentInfo) []UnaffectVulnerability { - affectBranchListx := strings.Split(affectBranch, "-") - cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", - affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + //affectBranchListx := strings.Split(affectBranch, "-") + //cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", + // affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + cpe := affectBranch if vulnerability != nil && len(vulnerability) > 0 { cveExist := false for _, vl := range vulnerability { - if vl.Cve == v.CveNum && vl.Remediations.Remediation.Description == v.InfluenceComponent { + if vl.Cve == v.CveNum && vl.Remediations != nil && len(vl.Remediations.Remediation) > 0 && + vl.Remediations.Remediation[0].Description == v.InfluenceComponent { cpeExist := false for _, pid := range vl.ProductStatuses.Status.ProductId { if pid.ProductId == cpe { @@ -405,6 +418,12 @@ func BuildUnaffVulnerabilitySlice(vulnerability []UnaffectVulnerability, v model var productId ProductId productId.ProductId = cpe vl.ProductStatuses.Status.ProductId = append(vl.ProductStatuses.Status.ProductId, productId) + var remediation UnRemediation + remediation.Type = "Unaffected" + remediation.Description = v.InfluenceComponent + remediation.Date = common.GetCurDate() + remediation.ProductId = cpe + vl.Remediations.Remediation = append(vl.Remediations.Remediation, remediation) } cveExist = true break @@ -458,11 +477,14 @@ func BuildUnaffVulnerability(vlLenth int, v models.ExcelExport, cVSSScoreSets.ScoreSet = &scoreSet vulnerability.CvssScoreSets = &cVSSScoreSets var remediations UnRemediations + remediationSlice := make([]UnRemediation, 0) var remediation UnRemediation remediation.Type = "Unaffected" remediation.Description = v.InfluenceComponent remediation.Date = common.GetCurDate() - remediations.Remediation = &remediation + remediation.ProductId = cpe + remediationSlice = append(remediationSlice, remediation) + remediations.Remediation = remediationSlice vulnerability.Remediations = &remediations vulnerabilitySlice = append(vulnerabilitySlice, vulnerability) return vulnerabilitySlice @@ -567,7 +589,7 @@ func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, te.Note = dSplit[0] + te.Note } if !strings.Contains(te.Note, dSplit[1]) { - te.Note += dSplit[1] + te.Note += dSplit[1] + "\r\n" } } te.Note = te.Note @@ -1040,9 +1062,10 @@ func BuildVulnerability(vlLenth int, v models.ExcelExport, func BuildVulnerabilitySlice(vulnerability []Vulnerability, v models.ExcelExport, affectBranch string, componentMap map[string]ComponentInfo) []Vulnerability { - affectBranchListx := strings.Split(affectBranch, "-") - cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", - affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + //affectBranchListx := strings.Split(affectBranch, "-") + //cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", + // affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) + cpe := affectBranch if vulnerability != nil && len(vulnerability) > 0 { cveExist := false for _, vl := range vulnerability { @@ -1106,6 +1129,64 @@ func BuildCvrfXml(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, BuildVulnerabilitySet(cvrfsa, v, affectBranch, componentMap) } +func QueryCveMd5(cveNum []string, OwnedComponent, openEulerSANum string, fixFlag int8) bool { + sort.Strings(cveNum) + cveStr := strings.Join(cveNum, ",") + cveStrx := cveStr + "," + OwnedComponent + fileMd5 := common.EncryptMd5(cveStrx) + var cfr models.CvrfSaRecord + cfr.Md5 = fileMd5 + tbErr := models.GetCvrfRecord(&cfr, "cur_md5") + if tbErr == nil { + return true + } else { + cfr.CreateTime = common.GetCurTime() + cfr.Md5 = fileMd5 + cfr.OpenEulerSANum = openEulerSANum + cfr.Status = 1 + cfr.IsExport = 1 + cfr.CveNum = cveStr + cfr.PackName = OwnedComponent + cfr.AffectFlag = fixFlag + num, iErr := models.InsertCvrfRecord(&cfr) + if iErr != nil || num == 0 { + logs.Error("InsertCvrfRecord, ", iErr) + } + return false + } +} + +func UpdateCvrfRecord(openeuler_sa_num string, isExport int8) { + var cfr models.CvrfSaRecord + cfr.UpdateTime = common.GetCurTime() + cfr.Status = 1 + cfr.IsExport = isExport + cfr.OpenEulerSANum = openeuler_sa_num + uErr := models.UpdateCvrfRecord(&cfr, "UpdateTime", "Status", "IsExport") + if uErr != nil { + logs.Error("UpdateCvrfRecord, ", uErr) + } +} + +func ProcCvrfFileName(fileName string) { + var cfr models.SaFileList + cfr.FileName = fileName + tbErr := models.GetCvrfFileName(&cfr, "file_name") + if tbErr == nil { + cfr.Status = 1 + cfr.UpdateTime = common.GetCurTime() + models.UpdateCvrfFileName(&cfr, "Status", "UpdateTime") + } else { + cfr.CreateTime = common.GetCurTime() + cfr.Status = 1 + cfr.FileName = fileName + num, iErr := models.InsertCvrfFileName(&cfr) + if iErr != nil || num == 0 { + logs.Error("InsertCvrfRecord, ", iErr) + } + } +} + func RecordCrvfInfo(fileName, filex string, fixFlag int8) error { fileBytes, err := ioutil.ReadFile(fileName) if err != nil { @@ -1135,17 +1216,22 @@ func RecordCrvfInfo(fileName, filex string, fixFlag int8) error { fileConStr := baseStdEncode(fileContent) var cfr models.CvrfSaRecord cfr.OpenEulerSANum = openEulerSANum - cfr.Md5 = fileMd5 + if fixFlag == UNAFFECTFLAG { + cfr.Md5 = fileMd5 + } + //cfr.Md5 = fileMd5 tbErr := models.GetCvrfRecord(&cfr, "openeuler_sa_num") if tbErr != nil || cfr.Id == 0 { cfr.CreateTime = common.GetCurTime() - cfr.Md5 = fileMd5 + if fixFlag == UNAFFECTFLAG { + cfr.Md5 = fileMd5 + cfr.CveNum = " " + cfr.PackName = " " + } cfr.OpenEulerSANum = openEulerSANum cfr.XmlContent = fileConStr cfr.Status = 1 cfr.IsExport = 1 - cfr.CveNum = " " - cfr.PackName = " " cfr.AffectFlag = fixFlag num, iErr := models.InsertCvrfRecord(&cfr) if iErr != nil || num == 0 { @@ -1153,7 +1239,9 @@ func RecordCrvfInfo(fileName, filex string, fixFlag int8) error { } } else { cfr.UpdateTime = common.GetCurTime() - cfr.UpdateMd5 = fileMd5 + if fixFlag == UNAFFECTFLAG { + cfr.Md5 = fileMd5 + } cfr.XmlContent = fileConStr cfr.Status = 1 cfr.IsExport = 1 @@ -1165,3 +1253,66 @@ func RecordCrvfInfo(fileName, filex string, fixFlag int8) error { } return nil } + +func PostFile(filename string, targetUrl string) error { + bodyBuf := &bytes.Buffer{} + bodyWriter := multipart.NewWriter(bodyBuf) + fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filename) + if err != nil { + logs.Error("error writing to buffer, ", err) + return err + } + fh, err := os.Open(filename) + if err != nil { + logs.Error("error opening file, ", err, ", filename: ", filename) + return err + } + _, err = io.Copy(fileWriter, fh) + if err != nil { + logs.Error(err) + return err + } + contentType := bodyWriter.FormDataContentType() + bodyWriter.Close() + resp, err := http.Post(targetUrl, contentType, bodyBuf) + if err != nil { + return err + } + defer resp.Body.Close() + resp_body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + logs.Info(resp.Status) + logs.Info(string(resp_body)) + return nil +} + + +func ReadWriteFile(filePath string, fileSlice []string) error { + if len(fileSlice) == 0 { + return errors.New("file content does not exist") + } + localDataSlice := make([]string, 0) + localFileSlice := Duplicate(fileSlice) + for _, lf := range localFileSlice { + localDataSlice = append(localDataSlice, lf.(string)) + } + sort.Strings(localDataSlice) + fileContent := strings.Join(localDataSlice, "\n") + fisExist, _ := PathExists(filePath) + if fisExist { + os.Remove(filePath) + } + file, err := os.Create(filePath) + if err != nil { + logs.Error("open file err", err) + return err + } + defer file.Close() + _, err = file.Write([]byte(fileContent)) + if err != nil { + logs.Error(err) + } + return nil +} \ No newline at end of file diff --git a/taskhandler/excel.go b/taskhandler/excel.go index a04d644..15b7bfa 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -544,6 +544,10 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, lz := len(list) if lz > 0 { for _, v := range list { + if v.OrganizateId == 2 { + logs.Error("opengauss, data: ", v) + continue + } affectBool := affectBrachRep(&v, affectBranch) if !affectBool { logs.Error("Unaffected version, data: ", v) @@ -669,19 +673,37 @@ func addXmlData(canExport []models.ExcelExport, cvexml *[]CveXml, affectBranch s } } +func GetOpenEulerSaNum() string { + var sa models.SaNumber + curYears := strconv.Itoa(time.Now().Year()) + models.GetSaNumberByYears(&sa, curYears, 1, 1) + if sa.SaId > 0 { + sa.Status = 2 + sa.UpdateTime = common.GetCurTime() + models.UpdateSaNumber(&sa, "status") + } + return sa.OpenEulerSANum +} + func StoreComponentInfo(componentMap map[string]ComponentInfo, v models.ExcelExport, affectBranch string) bool { repFlag := false influenceComponent := "" + fileDir := beego.AppConfig.String("fileDir") if len(v.InfluenceComponent) > 1 { influenceComponent = v.InfluenceComponent } else { influenceComponent = v.OwnedComponent } packNameStruct, pOk := componentMap[influenceComponent] - if !pOk && len(packNameStruct.OpenEulerSANum) < 1{ + if !pOk && len(packNameStruct.OpenEulerSANum) < 1 { var coponentInfo ComponentInfo - coponentInfo.OpenEulerSANum = v.OpenEulerSANum + openEulerSANum := GetOpenEulerSaNum() + if openEulerSANum == "" { + openEulerSANum = v.OpenEulerSANum + } + coponentInfo.OpenEulerSANum = openEulerSANum + coponentInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum+".xml") coponentInfo.OwnedComponent = influenceComponent coponentInfo.UpdateFlag = 2 openEulerScoreSlice := make([]float64, 0) @@ -689,10 +711,16 @@ func StoreComponentInfo(componentMap map[string]ComponentInfo, coponentInfo.OpenEulerScore = openEulerScoreSlice cveNumSlice := make([]string, 0) cveNumSlice = append(cveNumSlice, v.CveNum) - branchCve := make(map[string][]string) - branchCve[affectBranch] = cveNumSlice - //cveNum := make([]map[string][]string, 0) - //cveNum = append(cveNum, branchCve) + branchCve := make(map[string]BrachCveInfo) + var brachCveInfo BrachCveInfo + openEulerSANum1 := GetOpenEulerSaNum() + if openEulerSANum1 == "" { + openEulerSANum1 = v.OpenEulerSANum + } + brachCveInfo.OpenEulerSANum = openEulerSANum1 + brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") + brachCveInfo.CveNumSlice = cveNumSlice + branchCve[affectBranch] = brachCveInfo coponentInfo.CveNumMap = branchCve coponentInfo.CveNum = cveNumSlice componentMap[influenceComponent] = coponentInfo @@ -719,27 +747,45 @@ func StoreComponentInfo(componentMap map[string]ComponentInfo, packNameStruct.OpenEulerScore = openEulerScoreSlice } if len(packNameStruct.CveNumMap) > 0 { - if branchCve, brOk := packNameStruct.CveNumMap[affectBranch]; !brOk { + if branchCvex, brOk := packNameStruct.CveNumMap[affectBranch]; !brOk { cveNumSlice := make([]string, 0) cveNumSlice = append(cveNumSlice, v.CveNum) - packNameStruct.CveNumMap[affectBranch] = cveNumSlice + branchCve := make(map[string]BrachCveInfo) + var brachCveInfo BrachCveInfo + openEulerSANum1 := GetOpenEulerSaNum() + if openEulerSANum1 == "" { + openEulerSANum1 = v.OpenEulerSANum + } + brachCveInfo.OpenEulerSANum = openEulerSANum1 + brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") + brachCveInfo.CveNumSlice = cveNumSlice + branchCve[affectBranch] = brachCveInfo + packNameStruct.CveNumMap = branchCve } else { - for _, cve := range branchCve { + for _, cve := range branchCvex.CveNumSlice { if cve == v.CveNum { repFlag = true break } } if !repFlag { - branchCve = append(branchCve, v.CveNum) - packNameStruct.CveNumMap[affectBranch] = branchCve + branchCvex.CveNumSlice = append(branchCvex.CveNumSlice, v.CveNum) + packNameStruct.CveNumMap[affectBranch] = branchCvex } } } else { - branchCve := make(map[string][]string) + branchCve := make(map[string]BrachCveInfo) cveNumSlice := make([]string, 0) cveNumSlice = append(cveNumSlice, v.CveNum) - branchCve[affectBranch] = cveNumSlice + var brachCveInfo BrachCveInfo + openEulerSANum1 := GetOpenEulerSaNum() + if openEulerSANum1 == "" { + openEulerSANum1 = v.OpenEulerSANum + } + brachCveInfo.OpenEulerSANum = openEulerSANum1 + brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") + brachCveInfo.CveNumSlice = cveNumSlice + branchCve[affectBranch] = brachCveInfo packNameStruct.CveNumMap = branchCve } componentMap[influenceComponent] = packNameStruct @@ -767,18 +813,20 @@ func saveCvrfName(cvrfFileList map[string][]string, cvrffileName, mapKey string) } } -func procCvrfData(v models.ExcelExport, - affectBranch string, cvrfFileList map[string][]string, +func BranchCvrfData(v models.ExcelExport, + affectBranch, cvrffileName string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, - cvfrFileMap map[string]CvrfSa, pkg []models.Package) { - fileDir := beego.AppConfig.String("fileDir") - cvrffileName := filepath.Join(fileDir, "cvrf-"+componentMap[v.InfluenceComponent].OpenEulerSANum+".xml") - cvrfSaStruct, cvrfSaOk := cvfrFileMap[cvrffileName] - if !cvrfSaOk { + cvfrFileMap map[string]CvrfSa, pkg []models.Package, branchFlag int) { + branchCvrfSaStruct, bCvrfSaOk := cvfrFileMap[cvrffileName] + if !bCvrfSaOk { // Query whether there is data in the database var cfr models.CvrfSaRecord var cvrfsa CvrfSa - cfr.OpenEulerSANum = componentMap[v.InfluenceComponent].OpenEulerSANum + if branchFlag == 1 { + cfr.OpenEulerSANum = componentMap[v.InfluenceComponent].OpenEulerSANum + } else { + cfr.OpenEulerSANum = componentMap[v.InfluenceComponent].CveNumMap[affectBranch].OpenEulerSANum + } tbErr := models.GetCvrfRecord(&cfr, "openeuler_sa_num") if tbErr == nil && cfr.Id > 0 { // Download data from file server @@ -797,12 +845,22 @@ func procCvrfData(v models.ExcelExport, BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg) cvfrFileMap[cvrffileName] = cvrfsa } else { - BuildCvrfXml(&cvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg) - cvfrFileMap[cvrffileName] = cvrfSaStruct + BuildCvrfXml(&branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg) + cvfrFileMap[cvrffileName] = branchCvrfSaStruct } saveCvrfName(cvrfFileList, cvrffileName, CVRFFKEY) } +func procCvrfData(v models.ExcelExport, + affectBranch string, cvrfFileList map[string][]string, + componentMap map[string]ComponentInfo, + cvfrFileMap map[string]CvrfSa, pkg []models.Package) { + totalCvrffileName := componentMap[v.InfluenceComponent].CvrfFileName + branchCvrffileName := componentMap[v.InfluenceComponent].CveNumMap[affectBranch].CvrfFileName + BranchCvrfData(v, affectBranch, branchCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 2) + BranchCvrfData(v, affectBranch, totalCvrffileName, cvrfFileList, componentMap, cvfrFileMap, pkg, 1) +} + func (ec *CveExcel) setContentRow(v models.ExcelExport, affectBranch string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, cvfrFileMap map[string]CvrfSa) { diff --git a/taskhandler/hook.go b/taskhandler/hook.go index 6c64d3f..e75c86c 100644 --- a/taskhandler/hook.go +++ b/taskhandler/hook.go @@ -16,7 +16,7 @@ type HookData struct { ProjectId int64 } -func ProcHookEvent(hookurl, owner, accessToken, pwd string) error { +func ProcHookEvent(hookurl, owner, accessToken, pwd, gaussOwner, gitGaussToken string) error { hookId := int64(0) count := 5 for ; ; { @@ -24,6 +24,9 @@ func ProcHookEvent(hookurl, owner, accessToken, pwd string) error { if ihOk && len(iHook) > 0 { for _, ih := range iHook { var hd []HookData + if ih.Owner == gaussOwner { + accessToken = gitGaussToken + } GetDepositHooks(accessToken, ih.Owner, ih.Repo, &hd) if len(hd) > 0 { PrcMutDepositHooks(accessToken, pwd, ih, hd) diff --git a/taskhandler/issuestatistics.go b/taskhandler/issuestatistics.go new file mode 100644 index 0000000..5577f1f --- /dev/null +++ b/taskhandler/issuestatistics.go @@ -0,0 +1,156 @@ +package taskhandler + +import ( + "cvevulner/common" + "cvevulner/models" + "fmt" + "github.com/360EntSecGroup-Skylar/excelize/v2" + "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "os" + "path/filepath" + "strconv" +) + +const sheetName = "CVE_list" + +type IssueStr struct { + cveCount int64 + HighCveCount int64 + CurIndex int64 +} + +func GetIssueData(beforeDate, prcnum int, templateId int64) ([]models.IssueTemplate, error) { + beforeTime := common.GetSpecialDate(beforeDate) + it, err := models.QueryIssueTemplateByTime(beforeTime, prcnum, templateId, 3) + return it, err +} + +func createExcel() (string, string) { + // File storage directory + dir := beego.AppConfig.String("fileDir") + excelName := "CVE_ISSUE_" + common.GetCurDate() + ".xlsx" + excelPath := filepath.Join(dir, excelName) + //Determine whether the file exists + ok, _ := PathExists(excelPath) + if ok { + os.Remove(excelPath) + } + xlsx := excelize.NewFile() + index := xlsx.NewSheet(sheetName) + sheetTileMap := make(map[string]string) + sheetTileMap["A1"] = "序号" + sheetTileMap["B1"] = "issueID" + sheetTileMap["C1"] = "issue标题" + sheetTileMap["D1"] = "issue责任人" + sheetTileMap["E1"] = "issue状态" + sheetTileMap["F1"] = "issue创建时间" + sheetTileMap["G1"] = "issue所属仓库" + sheetTileMap["H1"] = "CVSS评分" + sheetTileMap["I1"] = "评分级别" + for k, v := range sheetTileMap { + xlsx.SetCellValue(sheetName, k, v) + } + xlsx.SetActiveSheet(index) + err := xlsx.SaveAs(excelPath) + if err != nil { + logs.Error(err) + } + return excelPath, dir +} + +func ReadWriteExcel(excelPath, owner string, iss []models.IssueTemplate, is *IssueStr) int64 { + file, openErr := excelize.OpenFile(excelPath) + if openErr != nil { + logs.Error("fail to open the file, ", excelPath) + return 0 + } + var templateId = int64(0) + for _, its := range iss { + templateId = its.TemplateId + cveData := procIssueData(its, is, owner) + if len(cveData) > 0 { + rows, sheetErr := file.GetRows(sheetName) + if sheetErr != nil { + logs.Error(sheetErr) + } + idx := len(rows) + 1 + axis := fmt.Sprintf("A%d", idx) + setErr := file.SetSheetRow(sheetName, axis, &cveData) + if setErr != nil { + logs.Error("setErr: ", setErr) + } + } + } + fileErr := file.SaveAs(excelPath) + if fileErr != nil { + logs.Error("Failed to save file, ", fileErr) + } + return templateId +} + +func procIssueData(its models.IssueTemplate, is *IssueStr, owner string) []interface{} { + cveData := make([]interface{}, 0) + if its.Owner != owner || len(its.IssueNum) < 2 { + return cveData + } + is.CurIndex += 1 + is.cveCount += 1 + cveData = append(cveData, is.CurIndex) + cveData = append(cveData, its.IssueNum) + cveData = append(cveData, its.CveNum) + cveData = append(cveData, its.Assignee) + cveData = append(cveData, IssueStateConversion(its.Status)) + cveData = append(cveData, common.TimeConverStr(its.CreateTime.String()[:19])) + cveData = append(cveData, its.Owner+"/"+its.OwnedComponent) + cveData = append(cveData, its.NVDScore) + cveData = append(cveData, openEulerScoreProc(its.NVDScore)) + if its.NVDScore >= 7 { + is.HighCveCount += 1 + } + return cveData +} + +func pressFileZip(excelPath, dir string) string { + totalFileSlice := make([]string, 0) + totalFileSlice = append(totalFileSlice, excelPath) + zipFileName := "CVE_ISSUE_" + common.GetCurDate() + ".zip" + zipFileName = filepath.Join(dir, zipFileName) + zipErr := ZipFiles(zipFileName, totalFileSlice, dir, dir) + if zipErr != nil { + logs.Error("File compression failed: err: ", zipErr) + } + return zipFileName +} + +func IssueStatistics(beforeDate, prcnum int, owner string) error { + excelPath, _ := createExcel() + fileSlice := make([]string, 0) + templateId := int64(0) + var is IssueStr + for { + it, err := GetIssueData(beforeDate, prcnum, templateId) + if err != nil { + return err + } + logs.Info("it==>", it) + if len(it) == 0 { + break + } + templateId = ReadWriteExcel(excelPath, owner, it, &is) + } + //zipFileName := pressFileZip(excelPath, dir) + zipFileName := excelPath + cBody := fmt.Sprintf("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, 10) + "个, 其中" + + strconv.FormatInt(is.HighCveCount, 10) + "个7分以上漏洞, 详情见附件, 请在20号之前解决, 优先解决7分以上CVE. \r\n" + + "已经分析完毕的issue请maintainer尽快关掉, 否则影响数据统计; 提交PR时要关联issue, 若CVE在之前PR解决, 请PR提交人编辑PR信息将issue关联上. \r\n") + sendError := SendEmail(zipFileName, 2, cBody) + if sendError != nil { + logs.Error("SendEmail, sendErr: ", sendError) + return sendError + } + fileSlice = append(fileSlice, excelPath) + fileSlice = append(fileSlice, zipFileName) + DelFile(fileSlice) + return nil +} diff --git a/taskhandler/oricvecheck.go b/taskhandler/oricvecheck.go index a4c41ec..b34ab76 100644 --- a/taskhandler/oricvecheck.go +++ b/taskhandler/oricvecheck.go @@ -4,7 +4,9 @@ import ( "cvevulner/common" "cvevulner/models" "github.com/astaxie/beego/logs" + "strconv" "strings" + "time" ) func CheckCveOriginData(prcnum int) (string, error) { @@ -41,9 +43,16 @@ func CheckCveOriginData(prcnum int) (string, error) { if len(pkList) == 2 { gits, ok := models.QueryCveOpeneulerdata(pkList[0], pkList[1]) if !ok { - models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 0) - logs.Info("仓库和版本对应关系错误, 还原: ", cveData, ", gits: ", gits) - break + opy := models.OpenGussYaml{PackageName: pkList[0], Version: pkList[1]} + openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") + if openErr == nil && opy.Id > 0 { + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) + logs.Info("加入到cve漏洞中: ", cveData, ", gits: ", gits) + } else { + models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 0) + logs.Info("仓库和版本对应关系错误, 还原: ", cveData, ", gits: ", gits) + break + } } else { models.UpdateOriginExist(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 1) logs.Info("加入到cve漏洞中: ", cveData, ", gits: ", gits) @@ -71,7 +80,7 @@ func UpdateAbnCveStatus(prcNum, days int, cveSt string) (string, error) { cveId = cd.CveId issueTmp := models.IssueTemplate{CveNum: cd.CveNum, CveId: cd.CveId, OwnedComponent: cd.PackName} err := models.GetIssueTemplateByColName(&issueTmp, "CveNum", "CveId", "OwnedComponent") - if issueTmp.TemplateId < 1{ + if issueTmp.TemplateId < 1 { cd.Status = 0 update := models.UpdateVulnCenter(&cd, "Status") if !update { @@ -100,3 +109,42 @@ func UpdateAbnCveStatus(prcNum, days int, cveSt string) (string, error) { } return "", nil } + +func AddSaNumber(saReAmount int64) (string, error) { + totalSaReAmount := 2 * saReAmount + curYears := strconv.Itoa(time.Now().Year()) + saNum := int64(1001) + models.DeleteSaNumberByYears(curYears, 1) + var count int64 + count = models.GetSaNumCountByYears(curYears, 1) + if count <= saReAmount { + var sa models.SaNumber + getErr := models.GetSaNumberByYears(&sa, curYears, 0, 2) + logs.Info("AddSaNumber, getErr: ", getErr) + if sa.SaId == 0 { + for i := int64(0); i < totalSaReAmount; i++ { + var localSa models.SaNumber + localSa.OpenEulerSANum = "openEuler-SA-" + curYears + "-" + strconv.FormatInt(saNum, 10) + localSa.Status = 1 + localSa.SaNum = saNum + localSa.SaYears = curYears + localSa.CreateTime = common.GetCurTime() + models.InsertSaNumber(&localSa) + saNum += 1 + } + } else { + saNum = sa.SaNum + 1 + for i := int64(0); i < totalSaReAmount-count; i++ { + var localSa models.SaNumber + localSa.OpenEulerSANum = "openEuler-SA-" + curYears + "-" + strconv.FormatInt(saNum, 10) + localSa.Status = 1 + localSa.SaNum = saNum + localSa.SaYears = curYears + localSa.CreateTime = common.GetCurTime() + models.InsertSaNumber(&localSa) + saNum += 1 + } + } + } + return "", nil +} diff --git a/taskhandler/sendemail.go b/taskhandler/sendemail.go index fb146d1..78f7b1b 100644 --- a/taskhandler/sendemail.go +++ b/taskhandler/sendemail.go @@ -87,7 +87,7 @@ func ZipFiles(filename string, files []string, oldform, newform string) error { return nil } -func SendEmail(attchStr string, flag int) error { +func SendEmail(attchStr string, flag int, cBody string) error { var mail Mail emailName := beego.AppConfig.String("email::email_name") emailPwd := beego.AppConfig.String("email::email_pwd") @@ -126,6 +126,41 @@ func SendEmail(attchStr string, flag int) error { } else { logs.Error("Notify cvrf mail delivery failure!") } + } else if flag == 2 { + ism, queryErr := models.QueryIssueStatisticEmail() + if len(ism) > 0 { + toEmail := make([]string, 0) + ccEmail := make([]string, 0) + for _, im := range ism { + if im.EmailType == 1 { + toEmail = append(toEmail, im.EmailName) + } else { + ccEmail = append(ccEmail, im.EmailName) + } + } + curDate := common.GetCurDateFormat() + message := Message{from: emailName, + to: toEmail, + cc: ccEmail, + bcc: []string{}, + subject: "【" + curDate + " CVE进展】openEuler需解决CVE列表，请各接口人尽快安排解决", + body: cBody, + contentType: "text/plain;charset=utf-8", + attachment: Attachment{ + name: attchStr, + contentType: "text/plain", + withFile: true, + }, + } + emailError = mail.Send(message) + if emailError == nil { + logs.Info("Notify issue statistics that the email was sent successfully!") + } else { + logs.Error("Notify issue statistics mail delivery failure!") + } + } else { + emailError = queryErr + } } else { message := Message{from: emailName, to: toEmailName, @@ -177,7 +212,8 @@ func (mail SendMail) Send(message Message) error { attachment := "\r\n--" + boundary + "\r\n" attachment += "Content-Transfer-Encoding:base64\r\n" attachment += "Content-Disposition:attachment\r\n" - attachment += "Content-Type:" + message.attachment.contentType + ";name=\"" + message.attachment.name + "\"\r\n" + attachment += "Content-Type:" + message.attachment.contentType + ";name=\"" + + strings.Replace(message.attachment.name, "download", "", -1) + "\"\r\n" buffer.WriteString(attachment) defer func() { if err := recover(); err != nil { @@ -187,7 +223,10 @@ func (mail SendMail) Send(message Message) error { mail.writeFile(buffer, message.attachment.name) } buffer.WriteString("\r\n--" + boundary + "--") - header := smtp.SendMail(mail.host+":"+mail.port, mail.auth, message.from, message.to, buffer.Bytes()) + toSend := make([]string, 0) + toSend = append(toSend, message.to...) + toSend = append(toSend, message.cc...) + header := smtp.SendMail(mail.host+":"+mail.port, mail.auth, message.from, toSend, buffer.Bytes()) logs.Info("header: ", header) return nil } diff --git a/util/parsepayload.go b/util/parsepayload.go index 1a892c1..9989097 100644 --- a/util/parsepayload.go +++ b/util/parsepayload.go @@ -9,6 +9,8 @@ import ( "sync" ) +const KwOpenGaussScore = "openGauss评分:" + const ( //KwAnalysisDesc 影响性分析说明的关键字 KwAnalysisDesc = "影响性分析说明:" -- Gitee From 1e29c720d259c365dd7fad6a273076cf15947ffa Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Mon, 18 Jan 2021 16:28:32 +0800 Subject: [PATCH 2/3] Format specification --- cve-py/downloadtask/downloadfiletask.py | 4 ++-- cve-py/tabletask/gauss_yaml.py | 25 +++++++++++++++---------- cve-py/tabletask/issue_statistics.py | 18 ++++++++++-------- 3 files changed, 27 insertions(+), 20 deletions(-) diff --git a/cve-py/downloadtask/downloadfiletask.py b/cve-py/downloadtask/downloadfiletask.py index 4921ed5..105e6f7 100644 --- a/cve-py/downloadtask/downloadfiletask.py +++ b/cve-py/downloadtask/downloadfiletask.py @@ -217,7 +217,7 @@ def download_excel(file_dir): """ local_dir = "./" + file_dir gitee_dir = 'https://gitee.com/openeuler/cve-manager/tree/master/cve-py/' + file_dir - urlx = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/%s/" %(file_dir) + urlx = "https://gitee.com/openeuler/cve-manager/raw/master/cve-py/%s/" % (file_dir) if os.listdir(local_dir): shutil.rmtree(local_dir) os.mkdir(local_dir) @@ -245,4 +245,4 @@ def download_excel(file_dir): code.write(r.content) if os.path.exists(local_dir + "/" + name): print("Download the excel successfully:" + name) - i += 2 \ No newline at end of file + i += 2 diff --git a/cve-py/tabletask/gauss_yaml.py b/cve-py/tabletask/gauss_yaml.py index c4f5a5a..39a8606 100644 --- a/cve-py/tabletask/gauss_yaml.py +++ b/cve-py/tabletask/gauss_yaml.py @@ -31,7 +31,8 @@ def download_gauss_yaml(): file_name = file_path + 'gauss_yaml.yaml' if mk_ok: # download the yaml file - file_url = 'https://gitee.com/opengauss/openGauss-third_party/raw/master/Third_Party_Open_Source_Software_List.yaml' + file_url = 'https://gitee.com/opengauss/openGauss-third_party' \ + '/raw/master/Third_Party_Open_Source_Software_List.yaml' downloadfiletask.download_gauss_yaml(file_name, file_url) return file_name @@ -63,16 +64,16 @@ def store_yaml_data(yaml_data): return None """ mysql = Mysql() - if yaml_data != None and len(yaml_data) > 0: + if yaml_data is not None and len(yaml_data) > 0: for yaml_key, yaml_value in yaml_data.items(): try: origin_data = select_yaml_origin_data(yaml_key, yaml_value, mysql) - if origin_data != None: + if origin_data is not None: update_yaml_origin_data(origin_data["id"], yaml_value, mysql) else: insert_yaml_origin_data(yaml_key, yaml_value, mysql) packages_data = select_yaml_data(yaml_key, yaml_value, mysql) - if packages_data != None: + if packages_data is not None: update_flag = False for pk in packages_data: if pk["package_id"] >= 10000000: @@ -83,7 +84,7 @@ def store_yaml_data(yaml_data): break if not update_flag: package_ids = select_yaml_lastdata() - if package_ids != None and package_ids["package_id"] >= 10000000: + if package_ids is not None and package_ids["package_id"] >= 10000000: packg_id = add_package_id(package_ids["package_id"]) else: packg_id = add_package_id(0) @@ -92,7 +93,7 @@ def store_yaml_data(yaml_data): mysql.end() else: package_ids = select_yaml_lastdata() - if package_ids != None and package_ids["package_id"] >= 10000000: + if package_ids is not None and package_ids["package_id"] >= 10000000: packg_id = add_package_id(package_ids["package_id"]) else: packg_id = add_package_id(0) @@ -138,7 +139,8 @@ def update_yaml_data(package_id, yaml_key, yaml_value, mysql): update data """ update_sql = "update cve_git_open_euler set origin_url = %s,update_time = %s, " \ - "cpe_packname = %s where package_id = %s and package_name =%s and version = %s" + "cpe_packname = %s where package_id = %s " \ + "and package_name =%s and version = %s" val = (yaml_value["url"], times.get_current_time(), yaml_value["cpeName"], package_id, yaml_key, yaml_value["version"]) mysql.update(update_sql, val) @@ -150,7 +152,8 @@ def update_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): """ update_sql = "update cve_git_package_info set package_name = %s,version = %s, " \ "origin_url = %s, update_time = %s where git_id = %s" - val = (yaml_key, yaml_value["version"], yaml_value["url"], times.get_current_time(), pg_detail_id) + val = (yaml_key, yaml_value["version"], yaml_value["url"], + times.get_current_time(), pg_detail_id) mysql.update(update_sql, val) @@ -160,7 +163,8 @@ def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): """ update_sql = "update cve_open_guss_yaml set origin_url = %s,status = %s, " \ "cpe_name = %s, update_time = %s where id = %s" - val = (yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), pg_origin_id) + val = (yaml_value["url"], 1, yaml_value["cpeName"], + times.get_current_time(), pg_origin_id) mysql.update(update_sql, val) mysql.dispose() @@ -184,7 +188,8 @@ def select_yaml_data(yaml_key, yaml_value, mysql): """ Query data """ - sql = "SELECT package_id, git_id FROM cve_git_open_euler WHERE package_name = %s " \ + sql = "SELECT package_id, git_id FROM cve_git_open_euler " \ + "WHERE package_name = %s " \ "and version = %s order by git_id desc" val = (yaml_key, yaml_value["version"]) packages_data = mysql.getMany(sql, val) diff --git a/cve-py/tabletask/issue_statistics.py b/cve-py/tabletask/issue_statistics.py index d9c6f08..15d1322 100644 --- a/cve-py/tabletask/issue_statistics.py +++ b/cve-py/tabletask/issue_statistics.py @@ -68,7 +68,7 @@ def get_issue_excel(): mysql = Mysql() files = os.listdir('./' + file_dir) for filename in files: - file_vaule = "./%s/%s" %(file_dir, filename) + file_vaule = "./%s/%s" % (file_dir, filename) with open(file_vaule, 'rb') as f: sha1obj = hashlib.sha1() sha1obj.update(f.read()) @@ -88,18 +88,19 @@ def get_issue_excel(): mysql.insertOne(sql, val) mysql.dispose() email_dict = proc_excel_data(file_vaule) - if email_dict != None and len(email_dict) > 0: + if email_dict is not None and len(email_dict) > 0: to_email_name_list = email_dict["to"] cc_email_name_list = email_dict["cc"] else: continue - if to_email_name_list != None and len(to_email_name_list) > 0: + if to_email_name_list is not None and len(to_email_name_list) > 0: del_sql = "delete from cve_issue_statistics_mail_list where email_type = %s" - mysql.delete(del_sql, (1, )) + mysql.delete(del_sql, (1,)) mysql.dispose() for email_name in to_email_name_list: try: - insert_sql = "insert into cve_issue_statistics_mail_list(email_name, email_type, create_time) values(%s,%s, %s)" + insert_sql = "insert into cve_issue_statistics_mail_list" \ + "(email_name, email_type, create_time) values(%s,%s, %s)" val = (email_name, 1, cur_date()) mysql.insertOne(insert_sql, val) mysql.dispose() @@ -107,12 +108,13 @@ def get_issue_excel(): print(e) mysql.dispose(2) del_sql = "delete from cve_issue_statistics_mail_list where email_type = %s" - mysql.delete(del_sql, (2, )) + mysql.delete(del_sql, (2,)) mysql.dispose() - if cc_email_name_list != None and len(cc_email_name_list) > 0: + if cc_email_name_list is not None and len(cc_email_name_list) > 0: for email_name in cc_email_name_list: try: - insert_sql = "insert into cve_issue_statistics_mail_list(email_name, email_type, create_time) values(%s,%s, %s)" + insert_sql = "insert into cve_issue_statistics_mail_list" \ + "(email_name, email_type, create_time) values(%s,%s, %s)" val = (email_name, 2, cur_date()) mysql.insertOne(insert_sql, val) mysql.dispose() -- Gitee From afb3ec8934bdb4194f091eac8e098285ed59a0c2 Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Mon, 18 Jan 2021 16:30:54 +0800 Subject: [PATCH 3/3] Format specification --- cve-py/tabletask/gauss_yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cve-py/tabletask/gauss_yaml.py b/cve-py/tabletask/gauss_yaml.py index 39a8606..5ba237c 100644 --- a/cve-py/tabletask/gauss_yaml.py +++ b/cve-py/tabletask/gauss_yaml.py @@ -237,7 +237,7 @@ def proc_gauss_yaml(): file_name = download_gauss_yaml() if files.file_isexists(file_name): yaml_data = parse_yaml(file_name) - if yaml_data != None and len(yaml_data) > 0: + if yaml_data is not None and len(yaml_data) > 0: store_yaml_data(yaml_data) else: print("File download failed, file: ", file_name) -- Gitee