diff --git a/common/aes.go b/common/aes.go index a1e530edc1b0fe565dd00fb4ed040e4da4d79dfa..bd7e5e465329518564a04cedcbb80134a16851f7 100644 --- a/common/aes.go +++ b/common/aes.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/aes" "crypto/cipher" + "crypto/md5" "encoding/base64" "errors" "flag" @@ -170,3 +171,30 @@ func GenToken(username, password string) (string, error) { tokens, err := setting(jwtkey, username, password) return tokens, err } + +////解析token +//func getting(tokenString string) (string, struct{}){ +// token, claims, err := ParseToken(tokenString) +// if err != nil || !token.Valid { +// return "", struct{}{} +// } +// return token, +//} +// +//func ParseToken(tokenString string) (*jwt.Token, *Claims, error) { +// Claims := &Claims{} +// token, err := jwt.ParseWithClaims(tokenString, Claims, func(token *jwt.Token) (i interface{}, err error) { +// return jwtkey, nil +// }) +// return token, Claims, err +//} + +//EncryptMd5 encrypt md5 +func EncryptMd5(str string) string { + if str == ""{ + return str + } + sum := md5.Sum([]byte(str)) + return fmt.Sprintf("%x",sum) +} + diff --git a/controllers/file.go b/controllers/file.go index 8d17494421ffb18c0ac638805efed97487cf2a01..1024b156572a8ad3d1e283d2da1e2f152dcc3ea2 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -1,8 +1,16 @@ package controllers import ( + "cvevulner/common" "cvevulner/models" + "cvevulner/taskhandler" + "cvevulner/util" + "fmt" "github.com/astaxie/beego" + "github.com/astaxie/beego/logs" + "path/filepath" + "regexp" + "time" ) //FileController file operation routing processing @@ -13,11 +21,83 @@ type FileController struct { //DownloadLastExcel Download the latest excel file // @router /lastExcel [get] func (f *FileController) DownloadLastExcel() { + fd := beego.AppConfig.DefaultString("fileDir", "download") + err := util.MakeDir(fd) + if err != nil { + logs.Error(err) + } er := models.ExportRecord{} - err := er.QueryLast() + err = er.QueryLast() if err != nil { _ = f.Ctx.Output.Body([]byte("no file ")) } - fp := "./" + er.FileName + fp := filepath.Join(fd, er.FileName) + if ex, _ := util.IsExistPath(fp); !ex { + _ = f.Ctx.Output.Body([]byte("no file ")) + } + f.Ctx.Output.Download(fp, er.FileName) +} + +//DownLoadExcelByFileCode download excel file by code +//@router /downloadExcel +func (f *FileController) DownLoadExcelByFileCode() { + fd := beego.AppConfig.DefaultString("fileDir", "download") + fc := f.GetString("fileCode") + if fc == ""{ + _ = f.Ctx.Output.Body([]byte("err: fileCode is a required parameter ")) + return + } + er := models.ExportRecord{FileCode: fc} + err := er.Read("file_code") + if err != nil { + _ = f.Ctx.Output.Body([]byte(fmt.Sprintf("err: %v",err))) + return + } + if er.FileName == ""{ + _ = f.Ctx.Output.Body([]byte("err: Can not find excel file by fileCode! ")) + return + } + if er.State == 0 { + _ = f.Ctx.Output.Body([]byte("The file is being generated, please try again later!")) + return + } + if er.State == 2 { + _ = f.Ctx.Output.Body([]byte("File generation failed, please contact the administrator or regenerate!!")) + return + } + fp := filepath.Join(fd, er.FileName) + if ex, _ := util.IsExistPath(fp); !ex { + _ = f.Ctx.Output.Body([]byte("error:file does not exist")) + } f.Ctx.Output.Download(fp, er.FileName) } + +//TriggerCveData touch off generate cve data excel and get cve package +//@router /triggerCveData [get] +func (f *FileController) TriggerCveData() { + startTime := f.GetString("startTime") + if startTime == "" { + f.Ctx.WriteString("Error:start time cannot be empty") + return + } + rt := regexp.MustCompile(`^(\d{4})-\d{2}-(\d{2})$`) + find := rt.Match([]byte(startTime)) + if !find { + f.Ctx.WriteString(`Error: please enter the correct start time in a format like this "yyyy-MM-dd".`) + return + } + now := time.Now().Unix() + en := fmt.Sprintf("cve与安全公告%v.xlsx", now) + fileCode := common.EncryptMd5(en) + //call the generate excel task by new thread + go taskhandler.GenerateExcelTrigger(en, startTime,fileCode) + er := models.ExportRecord{FileName: en, FileCode: fileCode, State: 0} + err := er.Insert() + if err != nil { + f.Ctx.WriteString(fmt.Sprintf("error:%v", err)) + } else { + //return the success notice to custom + f.Ctx.WriteString(fmt.Sprintf("Success：The name of the excel file generated this time is: %s. It takes some time to generate the excel file. "+ + "You can try to call the download file interface and pass in the param fileCode=%s to be downloaded.", en, fileCode)) + } +} diff --git a/doc/md/manual.md b/doc/md/manual.md index e857b0e246845e4a54d63f9b0e118dfba8b0176e..740e0c35f4a041b8617d069f7ca029955c9d137c 100644 --- a/doc/md/manual.md +++ b/doc/md/manual.md @@ -74,8 +74,42 @@ issue分析注意事项 异常关闭建议最好加上异常关闭原因同时建议使用指令而不是使用giee界面操作。) ``` -### 分析导出 -- 每天凌晨2：00 cve-manage 会通过定时任务生成excel文件 -- 通过 http://159.138.2.2:80/v1/download/excel 下载最新的excel文件 -- 未发布到官网的CVE 都会在最新的excel文档中 如果之前下载的未能及时同步到官网可下载最新文档再同步即可。 +### 数据导出 +#### 1.调用数据导出接口，触发生成excel的任务 + - 接口说明： + +| 说明 | 内容 | +| --- | ---- | +| 请求地址 | https://api.openeuler.org/v1/download/excel/triggerCveData | +| 请求类型 | GET | +- 参数： + +| 参数 | 是否必传 | 类型 | 数据类型 | 参数说明 | +| --- | --- | --- | --- | --- | +| startTime | yes | query | string | PR 合并的有效时间起始值 | + +- 响应返回： + ```batch +返回本次调用生成的excel文件名和fileCode +fileCode 用与下载本次调用生成的Excel文档 +``` +#### 2.下载cve与安全公告excel文档 + - 接口说明： + +| 说明 | 内容 | +| --- | ---- | +| 请求地址 | https://api.openeuler.org/v1/download/excel/downloadExcel | +| 请求类型 | GET | +- 参数： + +| 参数 | 是否必传 | 类型 | 数据类型 | 参数说明 | +| --- | --- | --- | --- | --- | +| fileCode | yes | query | string | 数据导出接口返回的fileCode | + +- 响应返回： + - 文件正在生成中： The file is being generated, please try again later! + - 文件生成完成 ： 返回excel文档 + - 文件生成失败： File generation failed, please contact the administrator or regenerate! + + diff --git a/main.go b/main.go index 29a77484f10ece2e9367dc26cef61ac1333361d4..2377e959d04809cddaa5e3301a6223ce52b61c00 100644 --- a/main.go +++ b/main.go @@ -2,6 +2,7 @@ package main import ( "cvevulner/common" + "cvevulner/models" _ "cvevulner/models" _ "cvevulner/routers" "cvevulner/task" @@ -13,13 +14,18 @@ func init() { common.InitGlobal() // Initialization log common.LogInit() - } func main() { + // init db + dbOk := models.Initdb() + if !dbOk { + println("error: Database initialization failed") + return + } // Initialize a scheduled task - ok := task.InitTask() - if !ok { + taskOk := task.InitTask() + if !taskOk { println("error: Timing task initialization failed, the program ends") return } diff --git a/models/cve.go b/models/cve.go index ab28d4e2a6be15abb9a388fe8927d433e722c61e..f6f7bee1b2df5f9b0503f2e0193c3132a2cfc8cf 100644 --- a/models/cve.go +++ b/models/cve.go @@ -2,6 +2,7 @@ package models import ( "cvevulner/common" + "errors" "fmt" "github.com/astaxie/beego/logs" "github.com/astaxie/beego/orm" @@ -668,6 +669,32 @@ ON b.cve_id = e.cve_id return } +//GetCanExportExcelData Get exportable data +func GetCanExportExcelData(cveNum,issueNum string) (list []ExcelExport, err error) { + if cveNum == "" { + return list, errors.New("cve number can not empty") + } + sql := `SELECT b.num,c.*,a.issue_num,a.owned_component,a.cve_brief, +d.sec_id,d.introduction,d.summary,d.theme,d.description,d.influence_component, +d.affect_product,d.reference_link,d.affect_status,e.public_date,openeuler_sa_num +FROM cve_issue_template a +RIGHT JOIN +(SELECT (SELECT COUNT(*) FROM cve_vuln_center WHERE cve_num = ?) num ,bc.cve_id,bc.cve_num +FROM cve_vuln_center bc WHERE bc.cve_num = ? AND bc.is_export = 3 ) b +ON a.cve_id = b.cve_id +LEFT JOIN cve_score c +ON b.cve_id = c.cve_id +LEFT JOIN cve_security_notice d +ON b.cve_id = d.cve_id +LEFT JOIN cve_open_euler_s_a e +ON b.cve_id = e.cve_id +WHERE a.issue_num = ? +` + o := orm.NewOrm() + _, err = o.Raw(sql, cveNum, cveNum,issueNum).QueryRows(&list) + return +} + func GetCanExportCveDataSameNum(cId string) (list []ExcelExport, err error) { sql := `SELECT a.cve_id,a.owned_component,a.cve_brief,c.*,d.sec_id,d.introduction,d.summary,d.theme,d.description,d.influence_component, d.affect_product,d.reference_link,d.affect_status,e.public_date,e.openeuler_sa_num diff --git a/models/excel.go b/models/excel.go index 3b5cb3d3778ca867d79e498025871d1037c46212..2cc21f1c7d35442c9ab0f426d83b843e5c11642b 100644 --- a/models/excel.go +++ b/models/excel.go @@ -8,7 +8,7 @@ type ExcelExport struct { Score CveBrief string `json:"cve_brief"` OwnedComponent string `json:"owned_component"` - SecID int64 `json:"sec_id"` + SecID int64 `json:"sec_id" orm:"column(sec_id)"` Introduction string `json:"introduction"` Summary string `json:"summary"` Theme string `json:"theme"` @@ -21,6 +21,13 @@ type ExcelExport struct { OpenEulerSANum string `json:"openeuler_sa_num" orm:"size(128);column(openeuler_sa_num)"` } +//ExcelPackage Released packages +type ExcelPackage struct { + PubTime string + Repo string + Packages string +} + //Insert Insert a generated excel file record func (er ExportRecord) Insert() error { o := orm.NewOrm() @@ -34,3 +41,14 @@ func (er *ExportRecord) QueryLast() error { err := o.QueryTable(er).OrderBy("-create_time").One(er) return err } +//Update update by column name +func (er *ExportRecord) Update(field ...string) error { + o := orm.NewOrm() + _, err := o.Update(er, field...) + return err +} +//Read read by column name +func (er *ExportRecord) Read(field ...string) error{ + o := orm.NewOrm() + return o.Read(er, field...) +} diff --git a/models/hookevent.go b/models/hookevent.go index 6bd774adaddff5dda937a83ad07a78b2dc3907ab..e48de652e04fb16c266517094a2b517cda8e2db8 100644 --- a/models/hookevent.go +++ b/models/hookevent.go @@ -41,7 +41,7 @@ type HookIssue struct { //IssueLabel issue label type IssueLabel struct { - Id string `json:"id"` + Id int64 `json:"id"` Name string `json:"name"` Color string `json:"color"` } @@ -105,6 +105,23 @@ type IssuePayload struct { Url string //issue URL on code cloud } +type PullRequest struct { + Id int64 + Number int + State string + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ClosedAt time.Time `json:"closed_at"` + MergedAt time.Time `json:"merged_at"` +} + +type PullRequestIssue struct { + Id int64 + Number string + Repo string + CveNumber string +} + //GetLabelsStr labels slice to string func (ih *HookIssue) GetLabelsStr() (labels string) { var lb []string diff --git a/models/initdb.go b/models/initdb.go index c51bde5730f0b772a0706c53c046687817b045a5..460d4f36b08e37b50994f60037dfffff172bb5a7 100644 --- a/models/initdb.go +++ b/models/initdb.go @@ -10,16 +10,16 @@ import ( _ "github.com/go-sql-driver/mysql" ) -func init() { - Initdb() -} +//func init() { +// Initdb() +//} //InitDb init database -func Initdb() { +func Initdb() bool{ BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { logs.Error("config init error:", err) - return + return false } //ConnDb() dbhost := BConfig.String("mysql::dbhost") @@ -43,12 +43,12 @@ func Initdb() { errx := orm.RegisterDriver("mysql", orm.DRMySQL) if errx != nil { logs.Error("RegisterDriver, orm err: ", errx) - return + return false } errorm := orm.RegisterDataBase("default", "mysql", dns, maxidle, maxconn) if errorm != nil { logs.Error("RegisterDataBase failed", "errorm: ", errorm) - return + return false } logs.Info("mysql 连接成功") res := CreateDb() @@ -56,7 +56,9 @@ func Initdb() { logs.Info("mysql table init success !") } else { logs.Error("mysql table init failed!") + return false } + return true } func ConnDb() (*sql.DB, error) { diff --git a/models/issue.go b/models/issue.go index 663a85727f5f9e630b1812ac20c6d4f1094b6626..7e78f58286aa4fdb83fb8af4326200ade555014f 100644 --- a/models/issue.go +++ b/models/issue.go @@ -124,9 +124,9 @@ func GetIssueTemplet(it *IssueTemplate) (localIt IssueTemplate, value bool) { } } -func GetIssueTemplateByColName(it *IssueTemplate, colName string) error { +func GetIssueTemplateByColName(it *IssueTemplate, colName ...string) error { o := orm.NewOrm() - err := o.Read(it, colName) + err := o.Read(it, colName...) return err } @@ -173,10 +173,44 @@ func UpdatePackageByCveId(pkgStr string, cveId int64) error { } } return nil + } return nil } +func ReplacePackageByCveId(pkgList []string,cveId int64) error { + //===== 先删除 再修改 ===== + sec := struct { + SecId int64 + }{} + secSql := `SELECT sec_id FROM cve_security_notice WHERE cve_id = ?` + o := orm.NewOrm() + err := o.Raw(secSql, cveId).QueryRow(&sec) + if err != nil { + return err + } + delPkgSql := `DELETE FROM cve_package WHERE sec_id = ?` + _, err = o.Raw(delPkgSql, sec.SecId).Exec() + if err != nil { + return err + } + pkgData := make([]Package, 0) + for _, v := range pkgList { + if strings.Trim(v," ")==""{ + continue + } + platform := "aarch64" + if strings.Contains(v,".x86_64."){ + platform = "x86_64" + } + pkgUrl := fmt.Sprintf(`https://repo.openeuler.org/openEuler-20.03-LTS/update/%s/Packages/%s`,platform, v) + pv := Package{SecId: sec.SecId, PackName: v, PackUrl: pkgUrl} + pkgData = append(pkgData, pv) + } + _, err = o.InsertMulti(1, pkgData) + return err +} + func QueryPackageByCveId(cveId int64) ([]Package, error) { sqlStr := `SELECT * FROM cve_package WHERE sec_id = (SELECT sec_id FROM cve_security_notice WHERE cve_id = ?)` var res []Package diff --git a/models/modeldb.go b/models/modeldb.go index ac15aa4d357f47256b206c1cbe28e7eecb0c7547..59bc3071d8e07e3d910603f0ab9ade9f4bef56bb 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -547,8 +547,10 @@ type GiteRepoBranch struct { } type ExportRecord struct { - Id int64 `orm:"pk;auto"` - FileName string `orm:"unique"` + Id int64 `orm:"pk;auto"` + FileName string `orm:"unique"` + FileCode string + State int8 ` description:"0:文件生成中;1 文件可下载；2 文件不可下载"` CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` } diff --git a/task/inittask.go b/task/inittask.go index b972d52620caf87ceaf4718792473728f23ee5e3..19a0cc174f78981f0e74ea77c21c7b13776a811c 100644 --- a/task/inittask.go +++ b/task/inittask.go @@ -149,6 +149,7 @@ func InitTask() bool { go InitYamlTask(getymal, &yamch) ok := <-yamch if !ok { + logs.Error("Failed to get yaml data: init task") return false } close(yamch) @@ -161,6 +162,7 @@ func InitTask() bool { go CheckOriCveTask(oricvecheck, &checkch) ok := <-checkch if !ok { + logs.Error("Filter cve data, whether it corresponds to the warehouse provided in yaml: init task") return false } close(checkch) @@ -178,6 +180,7 @@ func InitTask() bool { go InitIssueToken(issueoath, &ch) ok := <-ch if !ok { + logs.Error("Failed to get gitee token: init task") return false } close(ch) @@ -191,6 +194,7 @@ func InitTask() bool { go InitIssueTask(getissue, &issuech) ok := <-issuech if !ok { + logs.Error("Failed to synchronize template on gitee: init task") return false } close(issuech) @@ -203,6 +207,7 @@ func InitTask() bool { go InitCveTask(getcve, &cvech) ok := <-cvech if !ok { + logs.Error("Failed to initialize cve vulnerability library: init task") return false } close(cvech) @@ -215,6 +220,7 @@ func InitTask() bool { go PrintLogTask(printLog, &logch) ok := <-logch if !ok { + logs.Error("Failed to clean log: init task") return false } close(logch) @@ -227,6 +233,7 @@ func InitTask() bool { go CreatTask(createIssue, &ch) ok := <-ch if !ok { + logs.Error("Failed to create issue on gitee: init task", ) return false } close(ch) @@ -241,6 +248,7 @@ func InitTask() bool { go GenSAExcelTask(genExcel, &ch) ok := <-ch if !ok { + logs.Error("Exporting a closed issue from gitee failed: init task") return false } close(ch) diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 3dc8fa81c995799f69837e655021f0a149920421..38127facf825236929b0d567b80000fbddb229b6 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -3,14 +3,17 @@ package taskhandler import ( "cvevulner/common" "cvevulner/models" + "cvevulner/util" "encoding/json" "errors" "fmt" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" + "io" "io/ioutil" "net/http" "os" + "path/filepath" "strconv" "strings" "sync" @@ -18,7 +21,10 @@ import ( ) var GetCveDetailUrl = "http://cve.openeuler.org/cve-security-notice-server/cvedatabase/getByCveId?cveId=%s" -var lockx sync.Mutex +var ( + pkgLock sync.Mutex + lockx sync.Mutex +) var ewg sync.WaitGroup func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNum int, CveRes models.VulnCenter, @@ -1477,12 +1483,87 @@ func FilterCveExported() { func GenerateExcelTask() error { FilterCveExported() tn := time.Now().Format("2006-01-02") - fn := fmt.Sprintf("Cve数据与安全公告%s.xlsx", tn) - err := GenerateCveExcel(fn, "openEuler", 0, true) + dir := beego.AppConfig.DefaultString("fileDir", "download") + err := util.MakeDir(dir) if err != nil { return err } - er := models.ExportRecord{FileName: fn} + fn := filepath.Join(dir, fmt.Sprintf("Cve数据与安全公告%s.xlsx", tn)) + err = GenerateCveExcel(fn, "openEuler", 0, true) + if err != nil { + return err + } + er := models.ExportRecord{FileName: fn,FileCode: common.EncryptMd5(fn),State: 1} err = er.Insert() return err } +//GenerateExcelTrigger generate cve&security notice excel file by pr merge and influence package release. +func GenerateExcelTrigger(fileName, startTime,fileCode string) { + FilterCveExported() + dir := beego.AppConfig.DefaultString("fileDir", "download") + err := util.MakeDir(dir) + if err != nil { + logs.Error(err) + } + fr := models.ExportRecord{FileName: fileName} + err = fr.Read("file_name") + if err != nil { + logs.Error(err) + return + } + fileName = filepath.Join(dir, fileName) + du := "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" + du = beego.AppConfig.DefaultString("rpUrl",du) + localPath := filepath.Join(dir, "release-package.CSV") + err = downloadPackageFile(localPath, du) + if err != nil { + logs.Error(err) + fr.State = 2 + _ = fr.Update("state") + return + } + pkgList, err := ExtractPackageData(localPath) + if err != nil { + logs.Error(err) + fr.State = 2 + } else { + su := time.Now().Format("2006-01-02") + snPrefix := "openEuler-" + su + snSuffix := int64(1001) + err = GenerateCveExcelByTrigger(fileName, snPrefix, startTime, snSuffix, true, pkgList) + if err != nil { + logs.Error(err) + fr.State = 2 + } + fr.State = 1 + } + _ = fr.Update("state") +} + +func downloadPackageFile(localPath, url string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusOK { + pkgLock.Lock() + defer pkgLock.Unlock() + if ex, _ := util.IsExistPath(localPath); ex { + err := os.Remove(localPath) + if err != nil{ + logs.Error(err) + } + } + out, err := os.Create(localPath) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, resp.Body) + return err + } else { + return errors.New("download file request fail") + } + +} diff --git a/taskhandler/excel.go b/taskhandler/excel.go index fd6f913e15f4c08d5a69e5225c043411679be47c..484b8898a7b3ce845987cfc87b3f0c8360ed82cb 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -2,13 +2,22 @@ package taskhandler import ( "cvevulner/models" + "cvevulner/util" + "encoding/csv" + "encoding/json" "errors" "fmt" "github.com/360EntSecGroup-Skylar/excelize/v2" "github.com/astaxie/beego/logs" + "io" + "io/ioutil" + "net/http" "os" + "path" + "regexp" "strconv" "strings" + "sync" "time" ) @@ -28,6 +37,14 @@ type CveExcel struct { PackageURLSheetIndex int } +type IssueAndPkg struct { + IssueMap map[int64]models.PullRequestIssue + IssuePkg string +} + +var fillLock sync.Mutex +var wgTrigger sync.WaitGroup + //GenerateCveExcel Generate Excel documents based on data. //param snPrefix means security notice prefix. //param snSuffix means security notice suffix append start value. @@ -57,6 +74,29 @@ func GenerateCveExcel(excelName, snPrefix string, snSuffix int64, forceRewrite b return ec.Save(mode) } +//GenerateCveExcelByTrigger Generate cve&security notice excel file by trigger +func GenerateCveExcelByTrigger(excelName, snPrefix, startTime string, snSuffix int64, forceRewrite bool, pkgList []models.ExcelPackage) (err error) { + if len(pkgList) == 0 { + return errors.New("No data to export! ") + } + //init excelClient + ec := CveExcel{} + err = ec.Init(excelName, snPrefix, snSuffix) + if err != nil { + return err + } + mode := ec.InitFileHandle(forceRewrite) + if mode == 0 { + ec.InitSheet() + err = ec.FillHeader() + if err != nil { + logs.Error(err) + } + } + ec.FillContentTrigger(pkgList, startTime) + return ec.Save(mode) +} + //Init init excel client func (ec *CveExcel) Init(excelName, snPrefix string, snSuffix int64) (err error) { if excelName == "" || !(strings.HasSuffix(excelName, ".xlsx") || strings.HasSuffix(excelName, "xls")) { @@ -336,6 +376,32 @@ func (ec *CveExcel) FillContent(count int64) { } } +func (ec *CveExcel) FillContentTrigger(pkgList []models.ExcelPackage, startTime string) { + pl := len(pkgList) + pageSize := 10 + pc := pl / 10 + if pl%10 > 0 { + pc++ + } + cd := make(chan []IssueAndPkg) + start := 0 + end := 0 + for i := 0; i < pc; i++ { + start = i * pageSize + end = (i + 1) * pageSize + if end > pl { + end = pl + } + wgTrigger.Add(1) + go getDateByGite(pkgList[start:end], startTime, cd) + } + for i := 0; i < pc; i++ { + wgTrigger.Add(1) + go ec.handleGiteData(cd) + } + wgTrigger.Wait() +} + func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { list, err := models.GetCanExportCveData(off, size) if err != nil { @@ -392,6 +458,73 @@ func (ec *CveExcel) handleWriteContent(off int64, size int) (err error) { return nil } +func (ec *CveExcel) handleWriteContentSync(list []models.ExcelExport) (err error) { + lz := len(list) + if lz > 0 { + for _, v := range list { + if v.Num == 1 { + fillLock.Lock() + ec.setContentRow(v) + fillLock.Unlock() + } else if v.Num > 1 { + //处理CVE_NUM 重复 + //1.根据cve_num 获取issue_tpl 如果所有的issue_status ==2 0r issue_status == 6 则可以导出数据 + list, err := models.GetIssueTplByCveNum(v.CveNum) + if err != nil { + logs.Error(err) + continue + } + mergerList := make([]string, 0) + canMerger := true + for _, t := range list { + if t.Status != 3 && t.Status != 4 { + canMerger = false + break + } + if t.IssueStatus == 2 { + mergerList = append(mergerList, strconv.FormatInt(t.CveId, 10)) + } + + } + //2.从issue_status == 2 的模板数据中找到评分最高的对应的ExcelExport数据 + if canMerger && len(mergerList) > 0 { + canExport, err := models.GetCanExportCveDataSameNum(strings.Join(mergerList, ",")) + if err != nil { + logs.Error(err) + } + ep := canExport[0] + ep.SecID = v.SecID + if len(canExport) > 1 { + canExport = canExport[1:] + m := make(map[string]struct{}) + m[ep.OwnedComponent] = struct{}{} + for _, ex := range canExport { + //component repeat do not append + if _, ok := m[ex.OwnedComponent]; !ok { + ep.Introduction = ep.Introduction + "\n" + ex.Introduction + ep.Summary = ep.Summary + "\n" + ex.Summary + ep.Theme = ep.Theme + "\n" + ex.Theme + ep.Description = ep.Description + "\n" + ex.Description + ep.InfluenceComponent = ep.InfluenceComponent + "\n" + ex.InfluenceComponent + ep.AffectProduct = ep.AffectProduct + "\n" + ex.AffectProduct + ep.OwnedComponent = ep.OwnedComponent + "\n" + ex.OwnedComponent + m[ex.OwnedComponent] = struct{}{} + } + + } + + } + fillLock.Lock() + ec.setContentRow(ep) + fillLock.Unlock() + } + } + + } + } + return nil +} + func (ec *CveExcel) setContentRow(v models.ExcelExport) { pkg, err := models.GetCvePackageList(v.SecID) if err != nil { @@ -401,6 +534,44 @@ func (ec *CveExcel) setContentRow(v models.ExcelExport) { v.PublicDate = time.Now().Format("2006-01-02") } pkgStr := getPkgStr(pkg) + sn := []interface{}{v.OpenEulerSANum, v.CveNum, v.Introduction, v.Summary, v.Theme, v.Description, v.InfluenceComponent, + v.AffectProduct, pkgStr, v.ReferenceLink, v.PublicDate} + axis, searched := ec.searchValueInSheet(ec.SecNoticeSheetName, v.InfluenceComponent) + if !searched { + err = ec.fillSecurityNoticeSheet(sn) + if err != nil { + logs.Error(err) + } + } else { + //merge openEuler SA notice data + colReg := regexp.MustCompile(`[A-Z]*`) + col := colReg.FindString(axis) + row := strings.Trim(axis, col) + rCN := "B" + row + rRl := "J" + row + vcn, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rCN) + vcn += "\n" + v.CveNum + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rCN, vcn) + vrl, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rRl) + vrl += "\n" + v.ReferenceLink + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rRl, vrl) + rSAN := fmt.Sprintf("A%s", row) + vSAN, cellError := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rSAN) + if cellError == nil { + v.OpenEulerSANum = vSAN + } + //merger description + rd := "F" + row + vd, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rd) + dSplit := strings.Split(v.Description, "Security Fix(es):") + if len(dSplit) > 1 { + if !strings.Contains(vd, dSplit[0]) { + vd = dSplit[0] + vd + } + vd += dSplit[1] + } + _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rd, vd) + } cve := []interface{}{v.CveNum, v.CveBrief, v.NVDScore, v.OpenEulerScore, v.NattackVector, v.OattackVector, v.NattackComplexity, v.OattackComplexity, v.NprivilegeRequired, v.OprivilegeRequired, v.NuserInteraction, v.OuserInteraction, v.Nscope, v.Oscope, v.Nconfidentiality, v.Oconfidentiality, v.Nintegrity, v.Ointegrity, @@ -409,12 +580,7 @@ func (ec *CveExcel) setContentRow(v models.ExcelExport) { if err != nil { logs.Error(err) } - sn := []interface{}{v.OpenEulerSANum, v.CveNum, v.Introduction, v.Summary, v.Theme, v.Description, v.InfluenceComponent, - v.AffectProduct, pkgStr, v.ReferenceLink, v.PublicDate} - err = ec.fillSecurityNoticeSheet(sn) - if err != nil { - logs.Error(err) - } + ap := []interface{}{v.CveNum, v.AffectProduct, v.InfluenceComponent, v.AffectStatus} err = ec.fillAffectProductSheet(ap) if err != nil { @@ -422,9 +588,11 @@ func (ec *CveExcel) setContentRow(v models.ExcelExport) { } for _, v := range pkg { pk := []interface{}{v.PackName, v.PackUrl} - err := ec.fillPackageSheet(pk) - if err != nil { - logs.Error(err) + if _, ok := ec.searchValueInSheet(ec.PackageURLSheetName, v.PackName); !ok { + err := ec.fillPackageSheet(pk) + if err != nil { + logs.Error(err) + } } } } @@ -443,6 +611,28 @@ func getPkgStr(pkg []models.Package) string { return strings.Join(ps, ";\n") } +func (ec *CveExcel) searchValueInSheet(sheetName, value string) (axis string, searched bool) { + if value == "" { + return + } + sheet, err := ec.ExcelHandel.SearchSheet(sheetName, value) + if err != nil { + return + } + if len(sheet) > 0 { + for _, k := range sheet { + cellValue, _ := ec.ExcelHandel.GetCellValue(sheetName, k) + if cellValue == value { + searched = true + axis = k + break + } + } + return + } + return +} + func (ec *CveExcel) fillCveSheetRow(row []interface{}) (err error) { rows, err := ec.ExcelHandel.GetRows(ec.CveSheetName) if err != nil { @@ -495,3 +685,227 @@ func (ec *CveExcel) Save(md int8) error { return ec.ExcelHandel.Save() } + +//ExtractPackageData extract the package data by csv file +func ExtractPackageData(lp string) (pkgList []models.ExcelPackage, err error) { + pkgLock.Lock() + defer pkgLock.Unlock() + if lp == "" || path.Ext(lp) != ".CSV" { + return pkgList, errors.New("the file path is error") + } + file, err := os.Open(lp) + if err != nil { + return pkgList, err + } + defer file.Close() + reader := csv.NewReader(file) + for { + line, err := reader.Read() + if err == io.EOF { + break + } else if err != nil { + return pkgList, err + } + pkgList = append(pkgList, models.ExcelPackage{PubTime: line[0], Repo: line[1], Packages: line[2]}) + } + return +} + +func getDateByGite(pkgList []models.ExcelPackage, startTime string, c chan<- []IssueAndPkg) { + defer wgTrigger.Done() + //token := beego.AppConfig.String("gitee::git_token") + token := "8457c66db66955376519059b97e33dd1" + //owner := beego.AppConfig.String("gitee::owner") + owner := "src-openeuler" + st := util.TimeStrToInt(startTime, "2006-01-02") + chData := make([]IssueAndPkg, 0) + for _, v := range pkgList { + rt := util.TimeStrToInt(v.PubTime, "20060102 15-04-05") + prList := getRepoAllPR(token, owner, v.Repo, st, rt) + //get pull request related issue + repoIssue := make(map[int64]models.PullRequestIssue, 0) + for _, p := range prList { + getPRRelatedAllIssue(token, owner, v.Repo, st, rt, p.Number, repoIssue) + } + if len(repoIssue) > 0 { + chData = append(chData, IssueAndPkg{IssueMap: repoIssue, IssuePkg: v.Packages}) + } + } + c <- chData +} + +func (ec *CveExcel) handleGiteData(c <-chan []IssueAndPkg) { + defer wgTrigger.Done() + data := <-c + var pkgList []string + + for _, v := range data { + //parse package string to list + pkgList = strings.Split(v.IssuePkg, " ") + if len(pkgList) == 0 { + continue + } + for _, iv := range v.IssueMap { + tpl := models.IssueTemplate{IssueNum: iv.Number, Repo: iv.Repo} + err := models.GetIssueTemplateByColName(&tpl, "issue_num", "repo") + if err != nil { + logs.Error(err) + continue + } + err = models.ReplacePackageByCveId(pkgList, tpl.CveId) + if err != nil { + logs.Info(err) + continue + } + //save data to excel + el, err := models.GetCanExportExcelData(tpl.CveNum, tpl.IssueNum) + if err != nil { + logs.Error(err) + return + } + err = ec.handleWriteContentSync(el) + if err != nil { + logs.Error(err) + } + } + } +} + +func getRepoAllPR(token, owner, repo string, startTime, releaseTime int64) (prList []models.PullRequest) { + pageSize := 20 + pageCount := 1 + url := fmt.Sprintf("https://gitee.com/api/v5/repos/%s/%s/pulls", owner, repo) + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error(err) + return + } + q := req.URL.Query() + q.Add("access_token", token) + q.Add("sort", "created") + q.Add("state", "merged") + q.Add("per_page", strconv.Itoa(pageSize)) + for { + q.Del("page") + q.Add("page", strconv.Itoa(pageCount)) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error(err) + break + } + if resp.StatusCode == http.StatusOK { + pr := make([]models.PullRequest, 0) + read, err := ioutil.ReadAll(resp.Body) + if err != nil { + logs.Error(err) + break + } + resp.Body.Close() + err = json.Unmarshal(read, &pr) + if err != nil { + logs.Error(err) + break + } + for _, v := range pr { + ct := v.MergedAt.Local().Unix() + if ct >= startTime && ct <= releaseTime { + prList = append(prList, v) + } + } + if len(pr) < pageSize { + break + } + pageCount++ + } else { + resp.Body.Close() + break + } + } + return +} + +func getPRRelatedAllIssue(token, owner, repo string, startTime, releaseTime int64, num int, issueList map[int64]models.PullRequestIssue) { + if issueList == nil { + return + } + url := fmt.Sprintf(`https://gitee.com/api/v5/repos/%s/%s/pulls/%v/issues`, owner, repo, num) + pageSize := 20 + pageCount := 1 + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error(err) + return + } + q := req.URL.Query() + q.Add("access_token", token) + q.Add("per_page", strconv.Itoa(pageSize)) + for { + q.Del("page") + q.Add("page", strconv.Itoa(pageCount)) + req.URL.RawQuery = q.Encode() + resp, err := http.DefaultClient.Do(req) + if err != nil { + logs.Error(err) + break + } + if resp.StatusCode == http.StatusOK { + var il []models.HookIssue + read, err := ioutil.ReadAll(resp.Body) + resp.Body.Close() + if err != nil { + logs.Error(err) + break + } + err = json.Unmarshal(read, &il) + if err != nil { + logs.Error(err) + break + } + for _, v := range il { + d, ok := isLegallyIssue(v, startTime, releaseTime) + if ok { + issueList[d.Id] = d + } + } + if len(il) < pageSize { + break + } + pageCount++ + } else { + resp.Body.Close() + break + } + } +} + +func isLegallyIssue(i models.HookIssue, startTime int64, releaseTime int64) (pri models.PullRequestIssue, ok bool) { + if i.IssueType != IssueType || i.State != "closed" { + return + } + ft := i.FinishedAt.Unix() + if startTime > ft || ft > releaseTime { + return + } + tt := strings.Trim(i.Title, " ") + regCveNum := regexp.MustCompile(`(?mi)CVE-[\d]{1,}-([\d]{1,})$`) + if tt != "" && regCveNum.Match([]byte(tt)) { + ok = true + } else { + sm := util.RegexpCveNumber.FindAllStringSubmatch(i.Body, -1) + if len(sm) > 0 && len(sm[0]) > 0 { + val := sm[0][1] + tt = util.GetCveNumber(util.TrimString(val)) + if tt != "" && regCveNum.Match([]byte(tt)) { + ok = true + } + } + } + if ok { + pri.Id = i.Id + pri.Number = i.Number + pri.CveNumber = tt + pri.Repo = i.Repository.Path + } + return +} diff --git a/util/file.go b/util/file.go new file mode 100644 index 0000000000000000000000000000000000000000..b4274dfe003357c6f771e54bb70fa08a53523f22 --- /dev/null +++ b/util/file.go @@ -0,0 +1,30 @@ +package util + +import "os" + +//IsExistPath Determine whether the file path exists +func IsExistPath(path string) (exist bool,err error) { + _,err = os.Stat(path) + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +//MakeDir make dir +func MakeDir(path string) error { + exist,err := IsExistPath(path) + if err != nil{ + return err + } + if !exist { + err := os.Mkdir(path, os.ModePerm) + if err != nil { + return err + } + } + return nil +} diff --git a/util/time.go b/util/time.go new file mode 100644 index 0000000000000000000000000000000000000000..531b00c882651380962963750d0f4f4b84bef747 --- /dev/null +++ b/util/time.go @@ -0,0 +1,21 @@ +package util + +import ( + "github.com/astaxie/beego/logs" + "time" +) +//TimeStrToInt parse time string to unix nano +func TimeStrToInt(ts ,layout string) int64 { + if ts == ""{ + return 0 + } + if layout == ""{ + layout = "2006-01-02 15:04:05" + } + t, err := time.ParseInLocation(layout, ts,time.Local) + if err != nil { + logs.Error(err) + return 0 + } + return t.Unix() +}