From 653213a2a47abe0511cbcccdccb658390f516eb5 Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Wed, 3 Feb 2021 17:44:08 +0800 Subject: [PATCH] Generate cvrf data repair; update the issue statistics mailing list --- common/common.go | 11 ++ conf/app.conf | 14 +- conf/product_app.conf | 6 +- controllers/file.go | 60 ++++---- cve-py/issue_statistics/to_email.xlsx | Bin 10694 -> 12011 bytes models/cve.go | 3 +- models/excel.go | 6 + taskhandler/cve.go | 65 ++++++--- taskhandler/cvrf.go | 190 ++++++++++++++++++-------- taskhandler/excel.go | 32 +++-- taskhandler/issuestatistics.go | 4 +- taskhandler/sendemail.go | 17 ++- util/http.go | 1 - 13 files changed, 276 insertions(+), 133 deletions(-) diff --git a/common/common.go b/common/common.go index 6693f3a..52bd155 100644 --- a/common/common.go +++ b/common/common.go @@ -5,6 +5,7 @@ import ( "github.com/astaxie/beego" "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" + "os" "strings" "time" ) @@ -432,3 +433,13 @@ func GetSpecialDate(beforeDate int) string { } return startDate } + +func CreateDir(Dir string) error{ + _, err := os.Stat(Dir) + if err != nil{ + if os.IsNotExist(err){ + os.Mkdir(Dir, 0777) + } + } + return err +} diff --git a/conf/app.conf b/conf/app.conf index 218c759..c8a4ae4 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -22,8 +22,8 @@ labelFixed = "CVE/FIXED" labelUnFix = "CVE/UNFIXED" labeUnaffected = "CVE/UNAFFECTED" fileDir = "download" -# release package download url -rpUrl = "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" +# release package download url http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv +rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" [mysql] @@ -48,11 +48,11 @@ maxsize=204800 ymalflag = 2 getymal = 0 0 1 * * 1 cveflag = 2 -getcve = 0 4 16 * * * +getcve = 0 14 11 * * * oricveflag = 2 oricvecheck = 0 16 19 * * * getissueflag = 2 -getissue = 0 57 15 * * * +getissue = 0 50 10 * * * issueflag = 2 createissue = 0 7 16 * * * test = 0/10 * * * * * @@ -62,7 +62,7 @@ genexcelflag = 2 genexcel = 0 0/10 * * * * days = -30 prcnum = 50 -printlogflag = 1 +printlogflag = 2 printlog = 0 0/20 * * * * unlockflag = 2 unlock = 0 30 19 * * * @@ -75,7 +75,7 @@ exceptcve = 0 3 19 * * * gensaflag = 2 gensa = 0 3 19 * * * issuestatisticflag = 2 -issuestatistic = 0 53 15 * * * +issuestatistic = 0 11 10 * * * cvecredflag = 2 cvecredit = 0 53 14 * * * urgenttaskflag = 2 @@ -149,7 +149,7 @@ snsuffix = 1002 # Version package excel download address # example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; # openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv -v_pack_20_03_url = "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv;openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv" +v_pack_20_03_url = "openEuler-20.03-LTS@https;openEuler-20.03-LTS-SP1@https" # Time difference in different time zones sa_timestamp_zone = 28800 diff --git a/conf/product_app.conf b/conf/product_app.conf index 6ebdb55..6d95624 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -23,7 +23,7 @@ labelUnFix = "CVE/UNFIXED" labeUnaffected = "CVE/UNAFFECTED" fileDir = "download" # release package download url -rpUrl = "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" +rpUrl = "https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" @@ -76,7 +76,7 @@ exceptcve = 0 0 4 * * * gensaflag = 1 gensa = 0 1 0 * * * issuestatisticflag = 1 -issuestatistic = 0 35 8 * * * +issuestatistic = 0 1 10 * * 1,3,5 cvecredflag = 1 cvecredit = 0 0 5 * * * urgenttaskflag = 1 @@ -145,7 +145,7 @@ snsuffix = 1002 # Version package excel download address # example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; # openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv -v_pack_20_03_url = "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv;openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv" +v_pack_20_03_url = "openEuler-20.03-LTS@https://openEuler-20.03-LTS.csv;openEuler-20.03-LTS-SP1@https://openEuler-20.03-LTS-SP1.csv" # Time difference in different time zones sa_timestamp_zone = 28800 diff --git a/controllers/file.go b/controllers/file.go index 81af0e4..b4b9ea0 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -25,10 +25,7 @@ type FileController struct { // @router /lastExcel [get] func (f *FileController) DownloadLastExcel() { fd := beego.AppConfig.DefaultString("fileDir", "download") - err := util.MakeDir(fd) - if err != nil { - logs.Error(err) - } + err := common.CreateDir(fd) er := models.ExportRecord{} err = er.QueryLast() if err != nil { @@ -45,6 +42,7 @@ func (f *FileController) DownloadLastExcel() { //@router /downloadExcel func (f *FileController) DownLoadExcelByFileCode() { fd := beego.AppConfig.DefaultString("fileDir", "download") + common.CreateDir(fd) fc := f.GetString("fileCode") if fc == "" { _ = f.Ctx.Output.Body([]byte("err: fileCode is a required parameter ")) @@ -120,12 +118,13 @@ func (f *FileController) TriggerCveData() { cvrfFileList := make(map[string][]string) cvrfFileMap := make(map[string]taskhandler.CvrfSa) affectBranchSlice := make([]string, 0) - du := "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" + du := "openEuler-20.03-LTS@https://gitee.com/unsunghero/obs_pkg_rpms/raw/master/latest_rpm/openEuler-20.03-LTS.csv" //du := beego.AppConfig.String("excel::v_pack_20_03_url") du = beego.AppConfig.DefaultString("excel::v_pack_20_03_url", du) csvPathList := strings.Split(du, ";") if len(csvPathList) > 0 { for _, csvP := range csvPathList { + time.Sleep(time.Second) openBranchx := strings.Split(csvP, "@") if len(openBranchx) == 2 { affectBranch := openBranchx[0] @@ -143,7 +142,7 @@ func (f *FileController) TriggerCveData() { // return the success notice // Need to be executed sequentially wgCrvf.Add(1) - go taskhandler.GenerateExcelTrigger(&wgCrvf, en, startTime, fileCode, affectBranch, + taskhandler.GenerateExcelTrigger(&wgCrvf, en, startTime, fileCode, affectBranch, csvDownPath, dir, cvrfFileList, componentMap, cvrfFileMap) // Return the result first, continue processing the data f.Ctx.WriteString(fmt.Sprintf("Success：The name of the excel file generated this time is: %s. "+ @@ -167,6 +166,7 @@ func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, // taskhandler.WriteCvrfXml(cvrfKey, &cvrfVule) // } //} + common.CreateDir(dir) writeCvrfSlice := make([]string, 0) totalFileSlice := make([]string, 0) if len(componentMap) > 0 { @@ -183,12 +183,15 @@ func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, if isEque { cvrfFileName := componentMap[comKey].CvrfFileName cvrfVule := cvrfFileMap[cvrfFileName] - if !taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, - componentMap[comKey].OpenEulerSANum, taskhandler.FIXEDFLAGE) { - taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) - writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) - } else { - UpdateOpenEulerSaNumStatus(comValue.OpenEulerSANum) + md5Ok, tmpOpenEulerSANum := taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, + componentMap[comKey].OpenEulerSANum, taskhandler.FIXEDFLAGE) + taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) + writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) + if md5Ok { + UpdateOpenEulerSaNumStatus(tmpOpenEulerSANum) + var saf models.SaFileList + saf.FileName = "cvrf-" + tmpOpenEulerSANum + ".xml" + models.DeleteCvrfFileName(&saf, "FileName") } for _, br := range comValue.CveNumMap { UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) @@ -197,12 +200,15 @@ func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, for _, br := range comValue.CveNumMap { cvrfFileName := br.CvrfFileName cvrfVule := cvrfFileMap[br.CvrfFileName] - if !taskhandler.QueryCveMd5(br.CveNumSlice, componentMap[comKey].OwnedComponent, - br.OpenEulerSANum, taskhandler.FIXEDFLAGE) { - taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) - writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) - } else { - UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) + md5Ok, tmpOpenEulerSANum := taskhandler.QueryCveMd5(br.CveNumSlice, componentMap[comKey].OwnedComponent, + br.OpenEulerSANum, taskhandler.FIXEDFLAGE) + taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) + writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) + if md5Ok { + UpdateOpenEulerSaNumStatus(tmpOpenEulerSANum) + var saf models.SaFileList + saf.FileName = "cvrf-" + tmpOpenEulerSANum + ".xml" + models.DeleteCvrfFileName(&saf, "FileName") } } UpdateOpenEulerSaNumStatus(comValue.OpenEulerSANum) @@ -210,12 +216,15 @@ func GenAndUploadCvrf(cvrfFileMap map[string]taskhandler.CvrfSa, } else { cvrfFileName := componentMap[comKey].CvrfFileName cvrfVule := cvrfFileMap[cvrfFileName] - if !taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, - componentMap[comKey].OpenEulerSANum, taskhandler.FIXEDFLAGE) { - taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) - writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) - } else { - UpdateOpenEulerSaNumStatus(comValue.OpenEulerSANum) + md5Ok, tmpOpenEulerSANum := taskhandler.QueryCveMd5(componentMap[comKey].CveNum, componentMap[comKey].OwnedComponent, + componentMap[comKey].OpenEulerSANum, taskhandler.FIXEDFLAGE) + taskhandler.WriteCvrfXml(cvrfFileName, &cvrfVule) + writeCvrfSlice = append(writeCvrfSlice, cvrfFileName) + if md5Ok { + UpdateOpenEulerSaNumStatus(tmpOpenEulerSANum) + var saf models.SaFileList + saf.FileName = "cvrf-" + tmpOpenEulerSANum + ".xml" + models.DeleteCvrfFileName(&saf, "FileName") } for _, br := range comValue.CveNumMap { UpdateOpenEulerSaNumStatus(br.OpenEulerSANum) @@ -257,9 +266,10 @@ func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, d sfl := models.GetCvrfAllFile() indexFilePath := filepath.Join(dir, "index.txt") updateFilePath := filepath.Join(dir, "update_fixed.txt") + curYears := strconv.Itoa(time.Now().Year()) if len(sfl) > 0 { for _, l := range sfl { - totalSlice = append(totalSlice, l.FileName) + totalSlice = append(totalSlice, curYears+"/"+l.FileName) } //totalSlice = append(totalSlice, subFileSlice...) readErr := taskhandler.ReadWriteFile(indexFilePath, totalSlice) diff --git a/cve-py/issue_statistics/to_email.xlsx b/cve-py/issue_statistics/to_email.xlsx index 9d5ced369808b0b11e80745974ad2b0704fd1326..9575d691e239b399e5aebc2902998f8554c91e7e 100644 GIT binary patch delta 5837 zcmZvAcQ72_)Bl~`olcxyPKg@5_uis+POs6Tmm`Si<(w{hk9r6}5G{y)f{0$DOGpsq zmGAGJ-|u~Ae*4EW&$Ij4ot=4RcXmI+28UKnZ-FqC^(BBX5CBj{0{}<>0Dyy+y`Ha^ zw;!Lqm#-slkf%p-x|Qw#Bl+u7?OS5PKRivfOKRhrssTHsWX?jXBx1{NVoBw??n2Fu z50b2d)K}x;{(x(*Cxw=*R8-yOoqq)732| zK#$pddiQJV8iM?TmH;HW$MNr&!~_5~C;DTmzm* zC~AH1G!R}{#;$)+A?fNLO5A9*iSa&?Gl%_0RqunGeUcBfqLz8!YVSPgesb_VLrO5E zRAOyb?P82aGXeT@RyXd%JNDbX)5g<=f#R>-i$Ap}?8x3K(M5Fpz06*YuBAUT#G#AS zL*?_&f9Q@cm4K-$Paei{DMyJ)8tqta*|()0w2+1^6OAD`ta)_)#w) zWar9k+Prux@?7aY$ybMlS(mv1-|xx2EY8uthNO((oooIXK3g2(_-M(FsGZm0BSolM zxojCVlwH~u1`DDEXCQtZG;^*2E$w{%2*Phrynn!H(=h2hZ?UIDjy-A#3moq-n(Nf| zr?l&h?ql8QS6A94&w3J0Uc4tYE3YT!YF>_ESemG!z&|hbzcxuw^pwWVTYCWIsM~9SRtF6HkhWN_ddxtr;F* z+#BJ&PocJXEkDb?@1GE0ib{4bD<520F+KnlR|Bdwb=am}*u4{D+EO(oQB9tj3LIbF zr%BYXUJGwo6Zpl8HYI1zm-+5E28%9#_Df$N4LL2o_P?|{rBwB_~+e*-t+dA z0tfv+_cnAZU3r?1j8dqtQiCn&c>6&?NaH2i7G@;+NnOtz;I&TF-K?Fxr+|qi>?{}U z@6OQhja@_E`ZLVmnn}sR!tb@+C(&D{e;rYLB`D*GZj zo10g2xW%rtqV6*6?I#3GUjB>YI5PafzTs^B88GE;%za`2Jks#Z4Jr7qsS?b#fA8b$pHP_!?5fp>wOw97BWFYL8P)gElqjBTpR~ zR*V>HeJaWn;p=3+64WC#9+URvY0k_S-^@*pe2A;=CJhlq?uMKhY#^8i?}#WLC`X9} z$N?S&oimE__z5e@-O-}Cy!KW|&|N$_g7|9R6-EJE7~8mhWAdF*O~uF)zLf_xI=5P+ z*n*~qz>a#I1GCC9^yRjGG!_~9RgIp4VL`b`0 zYI>0T(@cDfy>*?im_OxJgkPsp&x1gJxV~rPk;=&zgIrvKr4x45$CecN?@0*01m&#BRfi?{+ zpFTU?7PsA(aw{6WL|w0KW_MBKFUKZS9<4xaawKjD*#iqR*|seJ+iU zRIs0Je2G>Pp9hJKZ++#Lr&mVXrp!0EsdN`np%p&Z!S|V)T&C-r9-VSC$CujRJBJ+)0LX^Rl2XD#QIkDNaJ?MwD`hU>B{$}7KB~=>-3Q>?)j3*x)Pg&G#N3lP3ia|W z3~DDEnM;4Nd>(u^x%Kuiv@rQl63h`aO3MN)ua&N~3UtXmDXPgqc!e*6dQ$zrm4?6?%-wTo!) z>G%zy?~O`vC38QN-?*$BUAVPb)4a5_S^0hzhaL)2)exn=@zE5X)5N>pU6$^suEX2- zN)%E&c0#*$HN*d=F+4de%%Jlpte)GXw#&;p|9bN`?N!}!MkCJglk2`#uj7w3#-zz& z=jLTHY&PmjTnhmkSHGV;k1U8!nrV95VLO=`rSzv+bDc z7U`+frsr^}fFUApONRWw5hZxYk)=F3x`!+PoHzy@S)QNgwYa=o6n(NhkB|ABe^QQw z4MX;03qOBrcmJuAoce_OB5a?SvsuqUN6N8TE!NEQ&Fzgc9$8ZX4=YvD4tS1Wxe5A) z14~sOnLyh99m9qBSynEA}D^nMl=%0%a#s79{wp{!8YHMIDvB&kOWgq zrmbwSG-*Eb6->1gGtF zUFQ%HkNQOH^PUxJ-&d6_-#qJznc#J&Lz^0RV-#hG&wF?e8ryz6a~cUG*(O7e!{ftu z+a|Z$#I~Z{kgkvRRKjO|o!0N_Q45#|p@#BztQ8$;yp+2yuNGlTEqG6AU!bPfg5#8> zMq^x$twFzS(^X3jxhcPJj8;q#PW>2}?u|LQ2i_z_E21Vae82?j#d`t2(R;W0p=>2; zGR~aP8XV(TbGNWa6aSbv2?sp9BeIjxi5}aIJ@Ml7?SRq|J7H4mSGEUizax>ageog{ z%ln#a4Sh+~2rOa6hSs$Gn1T?t5W<4DlHjMr!#s3%C4evZEUM|mDk+SRB=8VszZgZb zYvn)&VMHFbS)~5fr9y@@=9fgaM~LPlBnzx;P220BR*BRnlLqqBI<-N+UP$0Jbhp4e zvL%zAsbpN>yE+M2t1-(5GfhZYxOT>FuJ}uxvU9zIt({g;d{<(-t)8ma9T<}~KT_h} z*$2@Z&MN9RjE?TAjQIVzi`p6>ttA_NLB-P-uz_N#t5m(?YYJ^!EiDOJBh$;haBItpoKLX>Tr6H~7s~DAX z%@UtrErGS=>@K*pbo{S;d9owHOHAIJ1zPW+mUvb>z?!Jgahh~~PxTfoBF*x$uYDE{ zy!ms4ud9egt(oOIKX2E9EGO#9tX}%4Hg|dlaWC`71iTx?-VDb4=M+lh&6w-?}f0S(Wk6c;c&q!Bgqym|JfG66TVk z92s^!I%Kd%c6QaWbYgWy`aRVl*vkn0Vgjf+hNG{enoo2B?}c3PHMk-nS%8;BqpoZi z#>vUb%{l~TOazO|!C4J338BV@q=dpc%01n8ybBA|6@`aCb9u;jGn_Z$sHGSQgHUV=vOW$#gl zl!?{uyYHZTy&fBP3OYe;32`(!eV{|0(%T33>p`dzQ1OLxbVnnTvZZ`uc{^$b=?sUQ zvMb2ntWIC9Phh=b@=(ZS&$<0Oy1z6Pp?3#Wi2fj}61`@fS2WAZg~hN=jrd++0Q z`xiblr|QfA6T6=;1#~=@1TZt`7K;TSf)|IzxN{7sI296yd zDfKjcailJPy5!*V{+Z;5IsF3|zP~c!5xBE{zjwbhK()12PTx;RSe9{fHqT%K_^{R6 zTU4D0Ax^?M{Z#NZ@M5V7KKD>4EfId%-vy_svD;UJ(mfnDIzvt?n1`l;ZEVv{b{Ly3 zuM*B3)CTNsXGWJsV0!Du8mzBLgR8v1t&Yz1CPns|J~flywP`fsIqvgHQT?SL zGC(3r&?YBT(Ggk8GE8(to&jb ziee|EQcM#4SqGk*7c`(9cH%nNhVoZQM)l`mQ=u|)Sk86KnQSq3Xma)jqMJ#!$bP44 zTznpUyxM=wbSfDv5HgQa;8(?SsiaRzrbbqhfBo`veDRHdyM}?DosRie#$^n@@=mxb zkv)rwyY|60Ov{g&R>0hSanz#{qdxq5RJO!f!%pk_xT^#4yUFQ9uL_%X!+=}5TP*ae z!qv-^P{yl-CSPOE#sXfpmyxwfZts(cdfd$gd zV%c37+0)kJX<^j!?Vc0$>>|<8Bvr_7*JK>)g(yL(UNJ*s;Kp5c@mgTlaYY0OE8uBfFIP&a!#Cn${JogTguAFbdGrj)O6phKk}zt7CKo#V#TAs zsm}y^q&hn}i@GwfJJ%6QV_l$Xf~5@*mGx^(goPpD!5_`)xRi?NOm#mPm|}j#6R*#lr|iBV-HV~^!9ut?PY&w?dmFL5pzpLv z87;9L#h49nIhsOh@eTX5sv%mc|lV%cQ?k9vqIZ<)O@?;SUdAh zot1gn;{ljXRHj!{mVsou7qLwYFnoL>-31zItuX@wW=Z4Uy(G$!MF zP2+zID$tC)EP;qWr+FSf%5Jk;OU`L%Dly_tncpY0R4f?;)R z$XY$%BAw|BFo<|+-#t8gWY;PmKad3g@8n>E^_Wh$Ez5yEQr!^Qog%sqQO%0m7t@r! z`xWKH{D_5Fi8?cjI?r;caA)*bEHoY#>ZU%G-~09fAB)b9S5SX9v*}9lO}VWa457`q z*m0iq5mI4yO-9OKAn`i-H}eUR+OE65f$!Jq}wQTP)mb{!o!~X+NUT`SgwjBV~|k~ zA;O#u=#S;$EWvQPPe^;zMzF}TJ_>|Qpf@b(i3;jLuxFUHc^ZBakS`aTp|vEl^Htdx{NMkid1 zLw%Uq!h(97m3j>tJzHIf89}`wR}FQ`ajMT@PPoB`N)`%Pk;hr54GPad(Qze5q+M0! zU(6wKnhr+z*IZDKc#}LYWAjN9S9Ba%SL7x`Z(8$Y{z}M*O?7Kp)o%6XifdG<8E`ac2Q82`U%Hd#T|ogar)S8=%rSFL{2dxt6*~{`i=rM{&pL@e}>`S zSc>+$^G9`WD(Rp%yHA(;9~liC8#VRw%U3I!ab?q_J*LSscQ-4*xxe^~A`r`^HswXZ zHam+RMm#iu4PvyHfAj}4*rR>`o|5_Gb_Kp<8O|F$4AO@-Jhv&P^2U`v7R@>h_9!iY zK9je6Vg}=s{llqCLL>`K1bd3>Io_~AOr1P6qRH9EpApcqkH;@Y;97yVyb%YopQXR5 zJTM5L;164#gd9Y?{M&=NaZq{`e0-$*EL$jOIvr%rabPzd$dM6{?gRbf=eU5ZdiJMt z_e%@yB?(e67JFO%2azi+hg&bU)YW26O&V^#%u^~@Aeblfw6@UcXzt`*b!B`>~#oI^9tyj+FBLSYlm@pFm_%tc)4tkdJ z%BTFBYSwd8$pGH2NdDZs9+_iab=q8VB+u8WPSmT9Ij59ANadRiyW82RysUDQv;>!f z)b1fOin-Z3E)7K4afa!7vC*9mO-;n~4Kg#E#LZ3oNgiM)5qeXa z6l80t4$+oIHeaA40mt&qsVXmDCE>v)a>Srl_eALPv)T%O2{Vf^q|%!|+%58DbuPNE z=-e$P@ZJ%zPDwo~0iE-*X|v?u!?&kOgNtEl7V&2>{@=x6yY6{XJXjX-jeKK%nG^<^Vb~`jb&GwjMTS9tgV<(#5 z^!E|7pGN$%p>eLp(I7oSZzlI@(8lw3;d@_XsrUv8UN!hPA zY*?lq4n0h?5}BCTK2Z~b?*^7Y+`G``0SgB_6)PoG<-?N7{50_V&9PWt z{dYG74jn$1+>)U=U;^VyE*OdduGSCXJIqC8!-Q}ZmfL^~!$63S+^LUOM+RqgT-ac8 zfhsfY!HRQ-O7Z($@YlKy_0d#16o<;wWCso1%O{v*W6g+tYQ;z>+smhod7~$Gelm>8 z69{%IweK>*iq3qjr=RDRJv}~GG2G92p;INBTj8PFCgk%Cbl5ch^27&ZIZsUemDm9( zblt${S8Yc3z8W>YHjeXcoCuDnyFz%&k*qC}R*_4;8ZtO`pUAP?#^37(5tMzM^CJFB zV_mo2v>;pI^rG>=Pn|7y-uBYRRkHCC!3AmSeAZRVs!u9|iv_cf`W$zTKg||V^x~wH zGnqyQwFpG*0NDV^C(UxaVI7Uo5Fiw`c)Hir)q%|(c!k>zQR9^5(?*Od-q4t3hoTzd;w7~^@c%lXT>d$~bW4{tVvoWHv6`Jkz5Y5_Gyd4Z*Z28wu5#oG3- zm*pX)2+;a%NEsW{1driWXO9`KYMTYpMM0Mqx+d zezB`FQ<4Y`lp|}^Fvy{>jBra|(=+fzNqJgc!evmZNuaY^f*9t0c8ceu&bt)e;K?@Y ztxAGUI@eb%X9s_1!9cEiLtssYK7|5WMwN=ic)j(<(+rWyB9Lfh_|hHhowIFj!Byjz z_q@eU-s#0im24`S11Hd7zveqDS{?*aA|I_mw~5LRFa9>1Tc(BvtaOnkwRZ$JQ{^Ga zd?jjIv_Izr9Qa}86s1M&qnGJcbfXqD1VVCLFiR`8gd(FEYf+_Svr6+d#CXlUaSiK` zmHx=-T}oxoyvk-{A@W;8aCt3^D2L3n3A~;!@0;lLi$4IHlnDBwnV80gp+HMUNgE8hTqc&9&X8m2}ZJ z_ma)iIZb!tFi3>Cc9^|UvVdbB*Q_9WSL*D~HSstXObqUI_}@CynuebutsOpmu`w=&tn9qa4X5vI;<~xo7Kd`B+wvO;NKk(GEnvQpI4uB`e z9gSQ1w10mX_M$g$ISW5-dTq`fp_BSn9(8IsB~@{{MlPO1NhV5d zJm9JTpfp@NDy#Y0Jr(&917) z%xUiWJ<{-}OP}Egv4J+3kgKi0*2mrkK4(xGMJj~B<%Cx4OyBg!{cW)Pu^p>&^C_?S zO`|he+sD4$y3g_1?DzR&8+HQ^@c|8AFCzU{o2` zbMi)2GrdU(3#S;=Qs#!?N_Aqto~QD7y#&#tDD|;@9(Aj+#2JJ&W&MITo+vBlX_T#;7TOv z(GZvr@n3G6e8tHNhiPemMVm)OHe&Kn$FD6pJ{Siqx=2BC(=FM^&9-paUEz1JxYZx5 z`G^#5gS|9Rg-BGkYQS<%g(kZm<^d)4>FCfJ|A%g~!kp!{C~v)N2Gc=e1ylNX0cTGu zcZQ7TWK3E%)A{zdeFehXzS9f(LX2%=d=U`iu8^-!!c0Ln8MW{9eln=r#{4P+8)F3fdF%Y4 zga-DfgY2i&<})LP#hBpGb^F1cZyJYf>$@o3F$?M+*53k+Ew4+e;(rb|V3Q3YMNg|p z{@fz|4Zf!0J^LkYBtPk8TeXx_eO?sDhYST5`iB6WYRnkbi2Hw5dlwxe-gJJ{-yqH=i%#obj97IihXtPze(qc> znuCX1>_zE+-Nu(ggc9t*=nuIN#c==aUwfq8ju(WbA}T1IiQN$IZjc<1dT^W01p7y0 z;7B5--z3(NwbawZcd1mX+0sAyrd2sHz%Y6@uBqY`s(tmpHkW zNao=2+jM}j{bUiov$e9#QQ{x)@Rw%j;UUlWpVwbLe}4Jk&&tuH?)GzIuk`Yw`|B%? z7!`$+$K`3|#BULAhWBa8iUbAs%JIcLD?kxoeEF)g++44O_F*4-?ECI`2LldWJ zB;*JoSUQlcdJM{4I@2$_vnMWUHUjHh=I0RXy7l32il%^>99-x-?qhH_C2Nm(Sr}VH zRx9k#!yfXbQakzbZ0(4K?nU|Ca?!Jmtrg?vaa56VZ5Z@^|5f}g-Kjv9WrO%J&fAss zLuFXWAbPiaQo9m_a(d+ljfw#;7T1s{QbcVLhCONPLarwfjUaQdbpNok8x+#6jI%+4 zU(TH$4|*79#oy*?9xhT(U$r3<+!YDctY}6K!$u@8x<_2%iLD0|GZUhzZU7XlW#arn zZx*}z!i9xb^DqgqqpwBwlFs&fT^ok9yW_H)^0G3k4tH`D~lWV{u6jUpJW0kh9xx z#te)laD-r~`fTVcRx{7TgJ&LUb9Mi-?zG>*>@_gxpWkCylXHCl5Mj0dF^*^ z!fK<|$|gXFB}W+XuBE1BRWa60dLcS2Hd1qAEz0ZV2~$n}LUK{$grVi^jyiH;^Tc<1 zpDIB?o0tDwP=ayETkUb3i?;|A1913U{nTI%Xf@<*|K)VN>C5wvGbfis|J0E5SA3HU z6lX+Gf09ucH*_^qy<(#DAV5rn0Tc0j0eQtcKW#%7xR$<`2n%rQ%Q7B8wumTKZF0tP z@2l{)ZmuR_uM349pI&r&G%ZV4TG=bO2ZoK;+j7Os=`!c4wka>Dc=v!Ox^v0%+Bw&& zI^Nd|6?)^QS4~_WO2&4_lQ;LnO@osk8ge40I(KTh_m#Pk*@jb%RD2pdCRLhZJVpyX z76SROt&SPHnVUJHd~k`so~$;3h@ptQ?47H^#CGUdq>GF(M}x|j)IJc4EwqwQn?U8D!M zIZof%|9&5ei~-1CpGygyIpknVk%?U7ZW1gB-|TvO$sSJNQ&JuxC+_5&d^ocvS?gHw zo_LMFV!%+YQ>pcGHUn4FKsQ2pG--j6wBUbffdT(J2kK}9`iQ{4xq(?hHSh#Gz$zpO zK4J$@LaESyH79^&A#?B{7@#Ib0MeqYfS)iW5g!mCEDq-61geE)z`C4(5f7^f%76$= zf0aG|e@m?TO5iXqAc~g};1i-`{ZBMOpd0_T^516g|9}{PeqO5obGiR?1wbIi q|8*z`0;IqYfVS98@Lkd?l^J+0CImJk2Od)}1CwIZBv_t*@c#u7A5>=m diff --git a/models/cve.go b/models/cve.go index 0fc67d1..109707e 100644 --- a/models/cve.go +++ b/models/cve.go @@ -936,7 +936,8 @@ d.affect_product,d.reference_link,d.affect_status, e.public_date,e.openeuler_sa_num,a.cve_level,b.organizate_id FROM cve_issue_template a RIGHT JOIN -(SELECT (SELECT COUNT(*) FROM cve_vuln_center WHERE cve_num = ?) num ,bc.cve_id,bc.cve_num,bc.organizate_id +(SELECT (SELECT COUNT(*) FROM cve_vuln_center WHERE cve_num = ? AND is_export = 3) num , +bc.cve_id,bc.cve_num,bc.organizate_id FROM cve_vuln_center bc WHERE bc.cve_num = ? AND bc.is_export = 3 ) b ON a.cve_id = b.cve_id LEFT JOIN cve_score c diff --git a/models/excel.go b/models/excel.go index 9d144be..d4f3308 100644 --- a/models/excel.go +++ b/models/excel.go @@ -120,3 +120,9 @@ func UpdateCvrfFileName(afl *SaFileList, fields ...string) error { _, err := o.Update(afl, fields...) return err } + +func DeleteCvrfFileName(afl *SaFileList, fields ...string) error { + o := orm.NewOrm() + _, err := o.Delete(afl, fields...) + return err +} diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 6d4ed78..4a39365 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -4,13 +4,13 @@ import ( "cvevulner/common" "cvevulner/models" "cvevulner/util" + "encoding/base64" "encoding/json" "errors" "fmt" "github.com/astaxie/beego" "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" - "io" "io/ioutil" "net/http" "net/url" @@ -1983,7 +1983,7 @@ func GenerateExcelTrigger(wgx *sync.WaitGroup, fileName, startTime, fileCode, af } fileName = filepath.Join(dir, fileName) localPath := filepath.Join(dir, affectBranch+"_release-package.CSV") - err = downloadPackageFile(localPath, csvDownPath) + err = downloadPackageFile(localPath, csvDownPath, affectBranch) if err != nil { logs.Error(err) fr.State = 2 @@ -2007,7 +2007,6 @@ func GenerateExcelTrigger(wgx *sync.WaitGroup, fileName, startTime, fileCode, af fr.State = 1 } _ = fr.Update("state") - // zip file zipFileList := []string{fileName, filePath} //zipFileList := []string{"download/cve与安全公告openEuler-20.03-LTS_1607081986.xlsx", "download/updateinfo.xml"} @@ -2031,29 +2030,53 @@ func GenerateExcelTrigger(wgx *sync.WaitGroup, fileName, startTime, fileCode, af logs.Info(affectBranch, ", GenerateExcelTrigger: end") } -func downloadPackageFile(localPath, url string) error { - resp, err := http.Get(url) +func downloadPackageFile(localPath, url, affectBranch string) error { + logs.Info("The csv file currently being downloaded is: ", url) + pkgLock.Lock() + err := GiteeDownCsv(localPath, affectBranch) + pkgLock.Unlock() + return err +} + +func GiteeDownCsv(localPath, branch string) (error) { + accessToken := os.Getenv("GITEE_TOKEN") + owner := "unsunghero" + repo := "obs_pkg_rpms" + path := fmt.Sprintf("latest_rpm/%v.csv", branch) + gitUrl := fmt.Sprintf("https://gitee.com/api/v5/repos/%v/%v/contents/%v?access_token=%v", owner, repo, path, accessToken) + resp, err := http.Get(gitUrl) if err != nil { + logs.Error("get error, url:", gitUrl, "error: ", err) return err } defer resp.Body.Close() - if resp.StatusCode == http.StatusOK { - pkgLock.Lock() - defer pkgLock.Unlock() - if ex, _ := util.IsExistPath(localPath); ex { - err := os.Remove(localPath) - if err != nil { - logs.Error(err) - } - } - out, err := os.Create(localPath) - if err != nil { - return err - } - defer out.Close() - _, err = io.Copy(out, resp.Body) + body, err := ioutil.ReadAll(resp.Body) + if err != nil || body == nil { + logs.Error(err) + return err + } + logs.Info("body: \n", string(body), "url: ", gitUrl) + var contents map[string]interface{} + err = json.Unmarshal(body, &contents) + if err != nil { + logs.Error(err) return err + } + f, ferr := os.Create(localPath) + if ferr != nil { + logs.Error(ferr) + return ferr + } + fileType := contents["type"].(string) + encoding := contents["encoding"].(string) + content := contents["content"].(string) + if fileType == "file" && encoding == "base64" { + data, baseErr := base64.StdEncoding.DecodeString(content) + if baseErr == nil { + f.Write(data) + } } else { - return errors.New("download file request fail") + f.WriteString(content) } + return nil } diff --git a/taskhandler/cvrf.go b/taskhandler/cvrf.go index 83b8d02..5a8e21f 100644 --- a/taskhandler/cvrf.go +++ b/taskhandler/cvrf.go @@ -260,6 +260,7 @@ type BrachCveInfo struct { OpenEulerSANum string CvrfFileName string CveNumSlice []string + OpenEulerScore []float64 } type ComponentInfo struct { @@ -516,7 +517,7 @@ func BranchExist(affectBranch string, cvrfFileList map[string][]string) []string func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, introduction, topic, affectBranch string, componentMap map[string]ComponentInfo, - branchList []string) { + branchList []string, branchFlag int) { documentNotes := cvrfsa.DocumentNotes if documentNotes == nil || cvrfsa.DocumentNotes.Note == nil || len(cvrfsa.DocumentNotes.Note) == 0 { @@ -595,32 +596,53 @@ func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, te.Note = te.Note } if te.Title == "Topic" { - vcn := strings.Join(componentInfo.CveNum, ";\n") - theme, err := models.GetCanExportTheme(vcn, v.InfluenceComponent, affectBranch) - if err == nil && len(theme) > 1 { - theme = strings.ReplaceAll(theme, "\n\n", "\r\n\r\n") - if branchCount <= 1 { + vcn := "" + if branchFlag == 1 { + vcn = strings.Join(componentInfo.CveNum, ";\n") + theme, err := models.GetCanExportTheme(vcn, v.InfluenceComponent, affectBranch) + if err == nil && len(theme) > 1 { + theme = strings.ReplaceAll(theme, "\n\n", "\r\n\r\n") + if branchCount <= 1 { + te.Note = theme + } else if branchCount == 2 { + te.Note = strings.ReplaceAll(theme, affectBranch, strings.Join(branchList, " and ")) + } else { + reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] + te.Note = strings.ReplaceAll(theme, affectBranch, reBanch) + } + } + } else { + vcn = strings.Join(componentInfo.CveNumMap[affectBranch].CveNumSlice, ";\n") + theme, err := models.GetCanExportTheme(vcn, v.InfluenceComponent, affectBranch) + if err == nil && len(theme) > 1 { + theme = strings.ReplaceAll(theme, "\n\n", "\r\n\r\n") te.Note = theme - } else if branchCount == 2 { - te.Note = strings.ReplaceAll(theme, affectBranch, strings.Join(branchList, " and ")) - } else { - reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] - te.Note = strings.ReplaceAll(theme, affectBranch, reBanch) } } } if te.Title == "Summary" { - if branchCount <= 1 { - te.Note = v.Introduction - } else if branchCount == 2 { - te.Note = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) + if branchFlag == 1 { + if branchCount <= 1 { + te.Note = v.Introduction + } else if branchCount == 2 { + te.Note = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) + } else { + reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] + te.Note = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) + } } else { - reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] - te.Note = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) + te.Note = v.Introduction } + } if te.Title == "Severity" { - openEulerScoreSlice := componentInfo.OpenEulerScore + var openEulerScoreSlice []float64 + if branchFlag == 1 { + openEulerScoreSlice = componentInfo.OpenEulerScore + } else { + openEulerScoreSlice = componentInfo.CveNumMap[affectBranch].OpenEulerScore + } + if len(openEulerScoreSlice) > 1 { sort.Float64s(openEulerScoreSlice) } @@ -637,22 +659,38 @@ func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, func BuildDocumentTitle(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, - curDate string, branchList []string) { + curDate string, branchList []string, branchFlag int) { var documt DocumentTitle introduction := "" topic := "" branchCount := len(branchList) - if branchCount <= 1 { + ownedComponent := "" + if len(v.InfluenceComponent) > 1 { + ownedComponent = v.InfluenceComponent + } else { + ownedComponent = v.OwnedComponent + } + componentInfo := componentMap[ownedComponent] + localOpenEulerSANum := "" + if branchFlag == 1 { + localOpenEulerSANum = componentInfo.OpenEulerSANum + if branchCount <= 1 { + introduction = v.Introduction + topic = v.Theme + } else if branchCount == 2 { + introduction = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) + topic = strings.ReplaceAll(v.Theme, affectBranch, strings.Join(branchList, " and ")) + } else { + reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] + introduction = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) + topic = strings.ReplaceAll(v.Theme, affectBranch, reBanch) + } + } else { introduction = v.Introduction topic = v.Theme - } else if branchCount == 2 { - introduction = strings.ReplaceAll(v.Introduction, affectBranch, strings.Join(branchList, " and ")) - topic = strings.ReplaceAll(v.Theme, affectBranch, strings.Join(branchList, " and ")) - } else { - reBanch := strings.Join(branchList[:len(branchList)-1], ",") + " and " + branchList[len(branchList)-1] - introduction = strings.ReplaceAll(v.Introduction, affectBranch, reBanch) - topic = strings.ReplaceAll(v.Theme, affectBranch, reBanch) + localOpenEulerSANum = componentInfo.CveNumMap[affectBranch].OpenEulerSANum } + if len(introduction) > 1 { documt.DocumentTitle = introduction[:len(introduction)-1] } else { @@ -668,14 +706,7 @@ func BuildDocumentTitle(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch strin cvrfsa.DocumentPublisher = &documentPublisher var documentTracking DocumentTracking var identification Identification - ownedComponent := "" - if len(v.InfluenceComponent) > 1 { - ownedComponent = v.InfluenceComponent - } else { - ownedComponent = v.OwnedComponent - } - componentInfo := componentMap[ownedComponent] - identification.Id = componentInfo.OpenEulerSANum + identification.Id = localOpenEulerSANum documentTracking.Identification = &identification documentTracking.Status = "Final" documentTracking.Version = "1.0" @@ -713,11 +744,21 @@ func BuildDocumentTitle(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch strin documentTracking.Generator = &generator cvrfsa.DocumentTracking = &documentTracking BuilddocumentNotes(cvrfsa, v, introduction, topic, - affectBranch, componentMap, branchList) + affectBranch, componentMap, branchList, branchFlag) } -func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, componentMap map[string]ComponentInfo) { +func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, + componentMap map[string]ComponentInfo, branchFlag int, affectBranch string) { componentInfo, comOk := componentMap[v.InfluenceComponent] + localOpenEulerSANum := "" + localCveNum := []string{} + if branchFlag == 1 { + localOpenEulerSANum = componentInfo.OpenEulerSANum + localCveNum = componentInfo.CveNum + } else { + localOpenEulerSANum = componentInfo.CveNumMap[affectBranch].OpenEulerSANum + localCveNum = componentInfo.CveNumMap[affectBranch].CveNumSlice + } if comOk { if cvrfsa.DocumentReferences != nil && len(cvrfsa.DocumentReferences.CveReference) > 0 { cveReference := cvrfsa.DocumentReferences.CveReference @@ -758,14 +799,14 @@ func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, componentMap map[str var cveReference0 CveReference cveUrlSlice0 := make([]CveUrl, 0) var cveUrl0 CveUrl - cveUrl0.Url = "https://openeuler.org/en/security/safety-bulletin/detail.html?id=" + componentInfo.OpenEulerSANum + cveUrl0.Url = "https://openeuler.org/en/security/safety-bulletin/detail.html?id=" + localOpenEulerSANum cveUrlSlice0 = append(cveUrlSlice0, cveUrl0) cveReference0.Type = "Self" cveReference0.CveUrl = cveUrlSlice0 cveReferenceSlice = append(cveReferenceSlice, cveReference0) var cveReference1 CveReference cveUrlSlice1 := make([]CveUrl, 0) - for _, cveNum := range componentInfo.CveNum { + for _, cveNum := range localCveNum { var cveUrl1 CveUrl cveUrl1.Url = "https://openeuler.org/en/security/cve/detail.html?id=" + cveNum cveUrlSlice1 = append(cveUrlSlice1, cveUrl1) @@ -775,7 +816,7 @@ func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, componentMap map[str cveReferenceSlice = append(cveReferenceSlice, cveReference1) var cveReference2 CveReference cveUrlSlice2 := make([]CveUrl, 0) - for _, cveNum := range componentInfo.CveNum { + for _, cveNum := range localCveNum { var cveUrl2 CveUrl cveUrl2.Url = "https://nvd.nist.gov/vuln/detail/" + cveNum cveUrlSlice2 = append(cveUrlSlice2, cveUrl2) @@ -789,8 +830,9 @@ func BuildDocumentRef(cvrfsa *CvrfSa, v models.ExcelExport, componentMap map[str } } -func BuildProductTree(cvrfsa *CvrfSa, v models.ExcelExport, pkg []models.Package, affectBranch string) { - if cvrfsa.ProductTree != nil && len(cvrfsa.ProductTree.OpenEulerBranch) > 0 { +func BuildProductTree(cvrfsa *CvrfSa, v models.ExcelExport, pkg []models.Package, + affectBranch string, branchFlag int) { + if cvrfsa.ProductTree != nil && len(cvrfsa.ProductTree.OpenEulerBranch) > 0 && branchFlag == 1 { affectBranchListx := strings.Split(affectBranch, "-") cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) @@ -1004,7 +1046,7 @@ func BuildProductTree(cvrfsa *CvrfSa, v models.ExcelExport, pkg []models.Package } func BuildVulnerability(vlLenth int, v models.ExcelExport, - componentMap map[string]ComponentInfo, cpe string) []Vulnerability { + componentMap map[string]ComponentInfo, cpe string, branchFlag int) []Vulnerability { vulnerabilitySlice := make([]Vulnerability, 0) var vulnerability Vulnerability vulnerability.Xmlns = "http://www.icasi.org/CVRF/schema/vuln/1.1" @@ -1054,7 +1096,13 @@ func BuildVulnerability(vlLenth int, v models.ExcelExport, ownedComponent = v.OwnedComponent } componentInfo := componentMap[ownedComponent] - remediation.Url = "https://openeuler.org/en/security/safety-bulletin/detail.html?id=" + componentInfo.OpenEulerSANum + localOpenEulerSANum := "" + if branchFlag == 1 { + localOpenEulerSANum = componentInfo.OpenEulerSANum + } else { + localOpenEulerSANum = componentInfo.CveNumMap[cpe].OpenEulerSANum + } + remediation.Url = "https://openeuler.org/en/security/safety-bulletin/detail.html?id=" + localOpenEulerSANum remediations.Remediation = &remediation vulnerability.Remediations = &remediations vulnerabilitySlice = append(vulnerabilitySlice, vulnerability) @@ -1062,7 +1110,7 @@ func BuildVulnerability(vlLenth int, v models.ExcelExport, } func BuildVulnerabilitySlice(vulnerability []Vulnerability, v models.ExcelExport, - affectBranch string, componentMap map[string]ComponentInfo) []Vulnerability { + affectBranch string, componentMap map[string]ComponentInfo, branchFlag int) []Vulnerability { //affectBranchListx := strings.Split(affectBranch, "-") //cpe := fmt.Sprintf("cpe:/a:%v:%v:%v", // affectBranchListx[0], affectBranchListx[0], strings.Join(affectBranchListx[1:], "-")) @@ -1081,7 +1129,14 @@ func BuildVulnerabilitySlice(vulnerability []Vulnerability, v models.ExcelExport if !cpeExist { var productId ProductId productId.ProductId = cpe - vl.ProductStatuses.Status.ProductId = append(vl.ProductStatuses.Status.ProductId, productId) + if branchFlag == 1 { + vl.ProductStatuses.Status.ProductId = append(vl.ProductStatuses.Status.ProductId, productId) + } else { + productIdSlice := make([]ProductId, 0) + productIdSlice = append(productIdSlice, productId) + vl.ProductStatuses.Status.ProductId = productIdSlice + } + } cveExist = true break @@ -1089,21 +1144,21 @@ func BuildVulnerabilitySlice(vulnerability []Vulnerability, v models.ExcelExport } if !cveExist { vlLenth := len(vulnerability) + 1 - vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe) + vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe, branchFlag) if len(vulnerabilitySlice) > 0 { vulnerability = append(vulnerability, vulnerabilitySlice...) } } } else { vlLenth := 1 - vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe) + vulnerabilitySlice := BuildVulnerability(vlLenth, v, componentMap, cpe, branchFlag) vulnerability = append(vulnerability, vulnerabilitySlice...) } return vulnerability } func BuildVulnerabilitySet(cvrfsa *CvrfSa, v models.ExcelExport, - affectBranch string, componentMap map[string]ComponentInfo) { + affectBranch string, componentMap map[string]ComponentInfo, branchFlag int) { vulnerability := make([]Vulnerability, 0) if cvrfsa.Vulnerability != nil && len(cvrfsa.Vulnerability) > 0 { for _, vuln := range cvrfsa.Vulnerability { @@ -1111,27 +1166,28 @@ func BuildVulnerabilitySet(cvrfsa *CvrfSa, v models.ExcelExport, } } vulnerabilityx := BuildVulnerabilitySlice(vulnerability, v, - affectBranch, componentMap) + affectBranch, componentMap, branchFlag) cvrfsa.Vulnerability = vulnerabilityx } func BuildCvrfXml(cvrfsa *CvrfSa, v models.ExcelExport, affectBranch string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo, - pkg []models.Package) { + pkg []models.Package, branchFlag int) { curDate := common.GetCurDate() - v.Description = strings.ReplaceAll(v.Description, "\n\n", " ") - v.Theme = strings.ReplaceAll(v.Theme, "\n\n", " ") + //v.Description = strings.ReplaceAll(v.Description, "\n\n", " ") + //v.Theme = strings.ReplaceAll(v.Theme, "\n\n", " ") branchList := BranchExist(affectBranch, cvrfFileList) cvrfsa.Xmlns = "http://www.icasi.org/CVRF/schema/cvrf/1.1" cvrfsa.XmlnsCvrf = "http://www.icasi.org/CVRF/schema/cvrf/1.1" - BuildDocumentTitle(cvrfsa, v, affectBranch, cvrfFileList, componentMap, curDate, branchList) - BuildDocumentRef(cvrfsa, v, componentMap) - BuildProductTree(cvrfsa, v, pkg, affectBranch) - BuildVulnerabilitySet(cvrfsa, v, affectBranch, componentMap) + BuildDocumentTitle(cvrfsa, v, affectBranch, cvrfFileList, componentMap, curDate, branchList, branchFlag) + BuildDocumentRef(cvrfsa, v, componentMap, branchFlag, affectBranch) + BuildProductTree(cvrfsa, v, pkg, affectBranch, branchFlag) + BuildVulnerabilitySet(cvrfsa, v, affectBranch, componentMap, branchFlag) } -func QueryCveMd5(cveNum []string, OwnedComponent, openEulerSANum string, fixFlag int8) bool { +func QueryCveMd5(cveNum []string, OwnedComponent, openEulerSANum string, fixFlag int8) (bool, string) { sort.Strings(cveNum) + tmpOpenEulerSANum := "" cveStr := strings.Join(cveNum, ",") cveStrx := cveStr + "," + OwnedComponent fileMd5 := common.EncryptMd5(cveStrx) @@ -1139,7 +1195,21 @@ func QueryCveMd5(cveNum []string, OwnedComponent, openEulerSANum string, fixFlag cfr.Md5 = fileMd5 tbErr := models.GetCvrfRecord(&cfr, "cur_md5") if tbErr == nil { - return true + tmpOpenEulerSANum = cfr.OpenEulerSANum + cfr.UpdateTime = common.GetCurTime() + cfr.Md5 = fileMd5 + cfr.OpenEulerSANum = openEulerSANum + cfr.Status = 1 + cfr.IsExport = 1 + cfr.CveNum = cveStr + cfr.PackName = OwnedComponent + cfr.AffectFlag = fixFlag + updateErr := models.UpdateCvrfRecord(&cfr, "UpdateTime", + "OpenEulerSANum", "Status", "IsExport", "CveNum", "PackName", "AffectFlag") + if updateErr != nil { + logs.Error("UpdateCvrfRecord, ", updateErr) + } + return true, tmpOpenEulerSANum } else { cfr.CreateTime = common.GetCurTime() cfr.Md5 = fileMd5 @@ -1153,7 +1223,7 @@ func QueryCveMd5(cveNum []string, OwnedComponent, openEulerSANum string, fixFlag if iErr != nil || num == 0 { logs.Error("InsertCvrfRecord, ", iErr) } - return false + return false, tmpOpenEulerSANum } } diff --git a/taskhandler/excel.go b/taskhandler/excel.go index bfd875e..053c20c 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -516,6 +516,7 @@ func procUnaffectCvrfData(v models.ExcelExport, affectBranch string, cvrfFileList map[string][]string, componentMap map[string]ComponentInfo) { fileDir := beego.AppConfig.String("fileDir") + common.CreateDir(fileDir) cvrffileName := filepath.Join(fileDir, "cvrf-unaffected-cve-"+common.GetCurDate()+".xml") cvrfFileSlice, cvrfOk := cvrfFileList[UNAFFECTCVRFKEY] if cvrfOk && len(cvrfFileSlice) > 0 { @@ -544,6 +545,7 @@ func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport, lz := len(list) if lz > 0 { for _, v := range list { + logs.Info("handleWriteContentSync, v====>", v) if v.OrganizateId == 2 { logs.Error("opengauss, data: ", v) continue @@ -716,13 +718,14 @@ func StoreComponentInfo(componentMap map[string]ComponentInfo, repFlag := false influenceComponent := "" fileDir := beego.AppConfig.String("fileDir") + common.CreateDir(fileDir) if len(v.InfluenceComponent) > 1 { influenceComponent = v.InfluenceComponent } else { influenceComponent = v.OwnedComponent } packNameStruct, pOk := componentMap[influenceComponent] - if !pOk && len(packNameStruct.OpenEulerSANum) < 1 { + if !pOk || len(packNameStruct.OpenEulerSANum) < 3 { var coponentInfo ComponentInfo openEulerSANum := GetOpenEulerSaNum() if openEulerSANum == "" { @@ -746,6 +749,7 @@ func StoreComponentInfo(componentMap map[string]ComponentInfo, brachCveInfo.OpenEulerSANum = openEulerSANum1 brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") brachCveInfo.CveNumSlice = cveNumSlice + brachCveInfo.OpenEulerScore = openEulerScoreSlice branchCve[affectBranch] = brachCveInfo coponentInfo.CveNumMap = branchCve coponentInfo.CveNum = cveNumSlice @@ -776,7 +780,8 @@ func StoreComponentInfo(componentMap map[string]ComponentInfo, if branchCvex, brOk := packNameStruct.CveNumMap[affectBranch]; !brOk { cveNumSlice := make([]string, 0) cveNumSlice = append(cveNumSlice, v.CveNum) - branchCve := make(map[string]BrachCveInfo) + openEulerScoreSlice := make([]float64, 0) + openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) var brachCveInfo BrachCveInfo openEulerSANum1 := GetOpenEulerSaNum() if openEulerSANum1 == "" { @@ -785,8 +790,8 @@ func StoreComponentInfo(componentMap map[string]ComponentInfo, brachCveInfo.OpenEulerSANum = openEulerSANum1 brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") brachCveInfo.CveNumSlice = cveNumSlice - branchCve[affectBranch] = brachCveInfo - packNameStruct.CveNumMap = branchCve + brachCveInfo.OpenEulerScore = openEulerScoreSlice + packNameStruct.CveNumMap[affectBranch] = brachCveInfo } else { for _, cve := range branchCvex.CveNumSlice { if cve == v.CveNum { @@ -796,6 +801,7 @@ func StoreComponentInfo(componentMap map[string]ComponentInfo, } if !repFlag { branchCvex.CveNumSlice = append(branchCvex.CveNumSlice, v.CveNum) + branchCvex.OpenEulerScore = append(branchCvex.OpenEulerScore, v.OpenEulerScore) packNameStruct.CveNumMap[affectBranch] = branchCvex } } @@ -803,6 +809,8 @@ func StoreComponentInfo(componentMap map[string]ComponentInfo, branchCve := make(map[string]BrachCveInfo) cveNumSlice := make([]string, 0) cveNumSlice = append(cveNumSlice, v.CveNum) + openEulerScoreSlice := make([]float64, 0) + openEulerScoreSlice = append(openEulerScoreSlice, v.OpenEulerScore) var brachCveInfo BrachCveInfo openEulerSANum1 := GetOpenEulerSaNum() if openEulerSANum1 == "" { @@ -811,6 +819,7 @@ func StoreComponentInfo(componentMap map[string]ComponentInfo, brachCveInfo.OpenEulerSANum = openEulerSANum1 brachCveInfo.CvrfFileName = filepath.Join(fileDir, "cvrf-"+openEulerSANum1+".xml") brachCveInfo.CveNumSlice = cveNumSlice + brachCveInfo.OpenEulerScore = openEulerScoreSlice branchCve[affectBranch] = brachCveInfo packNameStruct.CveNumMap = branchCve } @@ -868,10 +877,12 @@ func BranchCvrfData(v models.ExcelExport, componentMap[v.InfluenceComponent] = componentInfo } } - BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg) + BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) cvfrFileMap[cvrffileName] = cvrfsa + logs.Info("xxxx1====>", cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg) } else { - BuildCvrfXml(&branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg) + BuildCvrfXml(&branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg, branchFlag) + logs.Info("xxxx2====>", branchCvrfSaStruct, v, affectBranch, cvrfFileList, componentMap, pkg) cvfrFileMap[cvrffileName] = branchCvrfSaStruct } saveCvrfName(cvrfFileList, cvrffileName, CVRFFKEY) @@ -1080,7 +1091,9 @@ func ExtractPackageData(lp string) (pkgList []models.ExcelPackage, err error) { } else if err != nil { return pkgList, err } - pkgList = append(pkgList, models.ExcelPackage{PubTime: line[0], Repo: line[1], Packages: line[2]}) + if len(line) >= 3 && len(line[0]) > 3 { + pkgList = append(pkgList, models.ExcelPackage{PubTime: line[0], Repo: line[1], Packages: line[2]}) + } } return } @@ -1098,6 +1111,7 @@ func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []I st := util.TimeStrToInt(startTime, "2006-01-02") chData := make([]IssueAndPkg, 0) for _, v := range pkgList { + logs.Info("excel, v===>", v) rt := util.TimeStrToInt(v.PubTime, "20060102 15-04-05") + saTimeStampZone // 查询当前需要处理的issue issueTemp, err := models.GetIssueNumber(v.Repo) @@ -1130,6 +1144,7 @@ func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg, affectBranch string, data := <-c var pkgList []string for _, v := range data { + logs.Info("v====>", v) //parse package string to list pkgList = strings.Split(v.IssuePkg, " ") if len(pkgList) == 0 { @@ -1206,7 +1221,7 @@ func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, mt := util.TimeStrToInt(closedAt[:19], "2006-01-02T15:04:05") mergedAt := v["merged_at"].(string) ct := util.TimeStrToInt(mergedAt[:19], "2006-01-02T15:04:05") - //logs.Info("******, pr: ", v, ",mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct) + logs.Info("******, pr: ", v, ",mt: ", mt, ", startTime: ", startTime, ",releaseTime: ", releaseTime, ":ct:", ct, ",repo:", repo) //ct := v["merged_at"].(string).(time.Time).Local().Unix() var pt int64 if mt > 0 && ct > 0 { @@ -1227,6 +1242,7 @@ func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, pr.CveNumber = isTemp.CveNum pr.Repo = repo prList = append(prList, pr) + logs.Info("prList====>", prList) } } } diff --git a/taskhandler/issuestatistics.go b/taskhandler/issuestatistics.go index 27c8528..6758728 100644 --- a/taskhandler/issuestatistics.go +++ b/taskhandler/issuestatistics.go @@ -29,6 +29,7 @@ func GetIssueData(beforeDate, prcnum int, templateId int64) ([]models.IssueTempl func createExcel() (string, string) { // File storage directory dir := beego.AppConfig.String("fileDir") + common.CreateDir(dir) excelName := "CVE_ISSUE_" + common.GetCurDate() + ".xlsx" excelPath := filepath.Join(dir, excelName) xlsx := excelize.NewFile() @@ -58,6 +59,7 @@ func createExcel() (string, string) { func CveCredExcel() (string, string) { // File storage directory dir := beego.AppConfig.String("fileDir") + common.CreateDir(dir) excelName := "CVE_Low_Credibility_" + common.GetCurDate() + ".xlsx" excelPath := filepath.Join(dir, excelName) xlsx := excelize.NewFile() @@ -93,7 +95,7 @@ func procCredData(org models.OriginUpstream, desc models.OriginUpstreamDesc) []i func ReadWriteExcel(excelPath, owner string, iss []models.IssueTemplate, is *IssueStr) (int64, error) { file, openErr := excelize.OpenFile(excelPath) if openErr != nil { - logs.Error("fail to open the file, ", excelPath) + logs.Error("fail to open the file, excelPath: ", excelPath, ", openErr: ", openErr) return iss[len(iss)-1].TemplateId, openErr } var templateId = int64(0) diff --git a/taskhandler/sendemail.go b/taskhandler/sendemail.go index 5c6623e..5697565 100644 --- a/taskhandler/sendemail.go +++ b/taskhandler/sendemail.go @@ -61,6 +61,11 @@ func ZipFiles(filename string, files []string, oldform, newform string) error { zipWriter := zip.NewWriter(newZipFile) defer zipWriter.Close() for _, file := range files { + fisExist, _ := PathExists(file) + if !fisExist { + logs.Error("ZipFiles, not exist, file: ", file) + continue + } zipfile, err := os.Open(file) if err != nil { return err @@ -108,12 +113,12 @@ func SendEmail(attchStr string, flag int, cBody, subject string) error { mail = &SendMail{user: emailName, password: emailPwd, host: emailHost, port: emailPort} if flag == 1 { message := Message{from: emailName, - to: toEmailName, - cc: []string{}, - bcc: []string{}, - subject: "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime(), - body: fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been " + - "uploaded to the file server is as follows: \r\n" + strings.ReplaceAll(attchStr,"download/", "")), + to: toEmailName, + cc: []string{}, + bcc: []string{}, + subject: "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime(), + body: fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been " + + "uploaded to the file server is as follows: \r\n" + strings.ReplaceAll(attchStr, "download/", "")), contentType: "text/plain;charset=utf-8", attachment: Attachment{ name: attchStr, diff --git a/util/http.go b/util/http.go index c94762e..fc8d023 100644 --- a/util/http.go +++ b/util/http.go @@ -195,7 +195,6 @@ func HTTPGet(url string) ([]map[string]interface{}, error) { //HTTPGetCom common get request func HTTPGetCom(urls string) ([]byte, error) { - logs.Info("httpgetcom: ", urls) resp, err := http.Get(urls) if err != nil { logs.Error("get error, url:", urls, "error: ", err) -- Gitee