diff --git a/cve-agency-manager/cve_tracking/conf/setting.py b/cve-agency-manager/cve_tracking/conf/setting.py index d8dccde58beb07ad148b7681afda2cbedf8256b7..16e8fd91e742b2aca1e36883ee2058c2ec288bdc 100644 --- a/cve-agency-manager/cve_tracking/conf/setting.py +++ b/cve-agency-manager/cve_tracking/conf/setting.py @@ -125,7 +125,7 @@ class YamlConfiguration: def _parse_yaml(self): with open(self.yaml, "r", encoding="utf-8") as file: try: - configs = yaml.load(file.read(), Loader=yaml.FullLoader) + configs = yaml.load(file.read(), Loader=yaml.SafeLoader) except yaml.YAMLError as error: raise ValueError( "The format of the yaml configuration " @@ -151,6 +151,7 @@ class YamlConfiguration: platform = filter(lambda x: x["name"] == name, self.platform) if platform: return list(platform)[-1] + return None def get_regex(self, label=None): """ @@ -168,6 +169,7 @@ class YamlConfiguration: token = filter(lambda x: x["name"] == name, self.authentication) if token: return list(token)[-1]["token"] + return None @property def configuration(self): diff --git a/cve-vulner-manager/Dockerfile b/cve-vulner-manager/Dockerfile index ba1d7bcd3ed6f3e63e1119b79ba585cf06aa49ba..386d3d3d430246c4bef5352c6c953f33ed6f4904 100644 --- a/cve-vulner-manager/Dockerfile +++ b/cve-vulner-manager/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.19.4 as BUILDER +FROM golang:1.24.1 as BUILDER LABEL maintainer="zhangjianjun" RUN go env -w GOPROXY=https://goproxy.cn,direct diff --git a/cve-vulner-manager/go.mod b/cve-vulner-manager/go.mod index 6b0c846583973e3b43c2c63b3f05f504f92f6eeb..89d733552b341dfdb33c77b3fad9ccdb9ff6f031 100644 --- a/cve-vulner-manager/go.mod +++ b/cve-vulner-manager/go.mod @@ -1,6 +1,6 @@ module cvevulner -go 1.19 +go 1.24.0 require ( github.com/antchfx/htmlquery v1.3.0 @@ -8,13 +8,14 @@ require ( github.com/dgrijalva/jwt-go v3.2.1-0.20210802184156-9742bd7fca1c+incompatible github.com/go-sql-driver/mysql v1.8.1 github.com/huaweicloud/huaweicloud-sdk-go-obs v3.23.4+incompatible + github.com/klauspost/compress v1.17.3 github.com/opensourceways/go-gitee v0.0.0-20240305060727-0df28a4f60c0 github.com/opensourceways/robot-gitee-lib v1.0.0 github.com/opensourceways/server-common-lib v0.0.0-20231027024402-f55c66e6699c github.com/robfig/cron/v3 v3.0.1 github.com/sirupsen/logrus v1.9.3 github.com/xuri/excelize/v2 v2.7.1 - golang.org/x/net v0.35.0 + golang.org/x/net v0.36.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/yaml.v2 v2.4.0 k8s.io/apimachinery v0.26.10 @@ -29,7 +30,6 @@ require ( github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect - github.com/klauspost/compress v1.17.3 // indirect github.com/lib/pq v1.8.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect @@ -44,7 +44,7 @@ require ( github.com/shiena/ansicolor v0.0.0-20151119151921-a422bbe96644 // indirect github.com/xuri/efp v0.0.0-20231025114914-d1ff6096ae53 // indirect github.com/xuri/nfp v0.0.0-20230919160717-d98342af3f05 // indirect - golang.org/x/crypto v0.33.0 // indirect + golang.org/x/crypto v0.35.0 // indirect golang.org/x/image v0.18.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sys v0.30.0 // indirect diff --git a/cve-vulner-manager/go.sum b/cve-vulner-manager/go.sum index d138219aeb79d6e2c52dacf9123e3772abb0ec01..a4fbef6261209e5e5d5ecccf5706b628589e906c 100644 --- a/cve-vulner-manager/go.sum +++ b/cve-vulner-manager/go.sum @@ -1091,8 +1091,8 @@ golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1221,8 +1221,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/cve-vulner-manager/obs/http.go b/cve-vulner-manager/obs/http.go index 9e501de9ee6e55a5d487dd5e0799d05436bc05c9..fa0d378e020e671f7dc630a52ab03d28e003b9ea 100644 --- a/cve-vulner-manager/obs/http.go +++ b/cve-vulner-manager/obs/http.go @@ -17,7 +17,6 @@ import ( "errors" "fmt" "io" - "math/rand" "net" "net/http" "net/url" @@ -489,7 +488,8 @@ func (obsClient ObsClient) doHTTP(method, bucketName, objectKey string, params m return nil, err } } - time.Sleep(time.Duration(float64(i+2) * rand.Float64() * float64(time.Second))) + const two = 2 + time.Sleep(time.Duration(float64(i+two) * float64(time.Second))) } else { doLog(LEVEL_ERROR, "Failed to send request with reason:%v", msg) if resp != nil { diff --git a/cve-vulner-manager/task/inittask.go b/cve-vulner-manager/task/inittask.go index f2c06ff86311555888611ac19f7bb28eeab9ca6d..74d0d8125a1419a194013a713d7ffaf3b6b7d6e7 100644 --- a/cve-vulner-manager/task/inittask.go +++ b/cve-vulner-manager/task/inittask.go @@ -6,7 +6,7 @@ import ( "github.com/astaxie/beego/toolbox" ) -//CheckOriCveTask Verify the original cve data +// CheckOriCveTask Verify the original cve data func CheckOriCveTask(oriCveCheck string) { logs.Info("The task of verifying the original cve data starts...") CheckTask := toolbox.NewTask("CheckOriCve", oriCveCheck, CheckOriCve) @@ -14,15 +14,7 @@ func CheckOriCveTask(oriCveCheck string) { logs.Info("End of verifying original cve data task...") } -//InitYamlTask Get yaml data source -func InitYamlTask(getYaml string) { - logs.Info("Get the yaml data source task started...") - yamlTask := toolbox.NewTask("GetYamlData", getYaml, GetYamlData) - toolbox.AddTask("GetYamlData", yamlTask) - logs.Info("End of the task of obtaining yaml data source...") -} - -//InitEulerYamlTask Get yaml data source +// InitEulerYamlTask Get yaml data source func InitEulerYamlTask(eulergetymal string) { logs.Info("Get the euleryaml data source task started...") eulerYamlTask := toolbox.NewTask("GetEulerYamlData", eulergetymal, GetEulerYamlData) @@ -78,7 +70,7 @@ func CreateHookTask(createHook string) { logs.Info("End of execution to create all webhook tasks...") } -//GenSAExcelTask Execute issue to generate excel task start +// GenSAExcelTask Execute issue to generate excel task start func GenSAExcelTask(genExcel string) { logs.Info("Execute issue to generate excel task start...") genExcelTask := toolbox.NewTask("GenExcelData", genExcel, GenExcelData) @@ -150,7 +142,7 @@ func PrintLogTask(printLog string) { logs.Info("Execute log task task end...") } -//Statistics of issues created in different communities, uncreated issues are created +// IssueCommunityStatistTask Statistics of issues created in different communities, uncreated issues are created func IssueCommunityStatistTask(issuecommunity string) { logs.Info("Community issue statistics task started...") issueStatTask := toolbox.NewTask("CommunityIssueStatist", issuecommunity, CommunityIssueStatist) @@ -158,7 +150,7 @@ func IssueCommunityStatistTask(issuecommunity string) { logs.Info("Community issue statistics task is over...") } -//Check whether the issue label is reasonable +// IssueLabelCheckTask Check whether the issue label is reasonable func IssueLabelCheckTask(issuelabelcheck string) { issueLaCheckTask := toolbox.NewTask("IssueLabelCheck", issuelabelcheck, IssueLabelCheck) toolbox.AddTask("IssueLabelCheck", issueLaCheckTask) @@ -192,19 +184,13 @@ func StopTask() { toolbox.StopTask() } -//InitTask Timing task initialization +// InitTask Timing task initialization func InitTask() bool { BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { logs.Error("config init error: file:conf/app.conf: ", err) return false } - // Get the original yaml data - ymalflag, errxs := BConfig.Int("crontab::ymalflag") - if ymalflag == 1 && errxs == nil { - getymal := BConfig.String("crontab::getymal") - InitYamlTask(getymal) - } // Verify the original cve data oricveflag, errxs := BConfig.Int("crontab::oricveflag") if oricveflag == 1 && errxs == nil { diff --git a/cve-vulner-manager/task/yaml.go b/cve-vulner-manager/task/yaml.go index 0bd70f1a0b4738191799efa1310b6f85c9bfb359..fd9261c8108106c216d46d338e4130242a650237 100644 --- a/cve-vulner-manager/task/yaml.go +++ b/cve-vulner-manager/task/yaml.go @@ -1,42 +1,14 @@ package task import ( - "cvevulner/common" - "cvevulner/taskhandler" "errors" + "github.com/astaxie/beego" - "github.com/astaxie/beego/config" "github.com/astaxie/beego/logs" -) -// GetYamlData Get yaml data source -func GetYamlData() error { - defer common.Catchs() - logs.Info("Get the yaml data source task started") - // Query the cve to be processed, 1: add; 2: modify - BConfig, err := config.NewConfig("ini", "conf/app.conf") - if err != nil { - logs.Error("GetYamlData, config init, error:", err) - return err - } - apiUrl := BConfig.String("yaml::apiurl") - if apiUrl == "" { - logs.Error("GetYamlData, config yaml::apiurl, error: invalid value") - return errors.New("value is nil") - } - // Get the data source of the table - _, err = taskhandler.GetYamlTables(apiUrl) - // Get yaml - if err == nil { - _, err = taskhandler.GetYamlByGit(apiUrl) - } - // Synchronize other sources of yaml version - taskhandler.SyncEulerYaml() - // Delete historical yaml source version data that does not exist - taskhandler.DelHistoryEulerYaml() - logs.Info("End of the task of obtaining yaml data source") - return err -} + "cvevulner/common" + "cvevulner/taskhandler" +) // GetEulerYamlData Get yaml data source func GetEulerYamlData() error { diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index 270a972993bffe160b59595fb568c3d64f20064c..73871c0c2b5bc3bb15947b61f0322f6a78b9ef21 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -93,7 +93,8 @@ func GetGiteeIssue(accessToken, owner, path, issueNum string) (error, map[string logs.Error(err) return err, respBody } - if respBody != nil && respBody["number"] != nil && respBody["number"].(string) == issueNum { + number, ok := respBody["number"] + if ok && number.(string) == issueNum { return nil, respBody } } else { diff --git a/cve-vulner-manager/taskhandler/yaml.go b/cve-vulner-manager/taskhandler/yaml.go index 41147be800abe167eeb5dd8fa8d4c8dfff07f8ef..ad46f4875da655f538c427bb3ab6e153ec0eaf40 100644 --- a/cve-vulner-manager/taskhandler/yaml.go +++ b/cve-vulner-manager/taskhandler/yaml.go @@ -2,11 +2,8 @@ package taskhandler import ( "encoding/json" - "errors" "fmt" "net/http" - "net/url" - "strconv" "strings" "sync" @@ -20,393 +17,6 @@ import ( var lock sync.Mutex -func GetYamlTables(url string) (string, error) { - compURL := url + "/lifeCycle/tables" - body, err := util.HTTPGetCom(compURL) - if err == nil && body != nil { - var respBody map[string]interface{} - err = json.Unmarshal(body, &respBody) - if err != nil { - logs.Error(err) - return "", err - } - logs.Info(respBody) - if respBody["code"].(string) == "2001" { - for _, values := range respBody["data"].([]interface{}) { - var gt models.GitPackageTable - gt.TableName = values.(string) - tableID, err := models.CreateYamlTable(>) - if tableID > 0 { - logs.Info("table: ", values, "Inserted successfully, table_id: ", tableID) - } else { - logs.Error("table: ", values, "Insertion failed, err: ", err) - return "", err - } - } - } else { - return "", errors.New("数据格式错误") - } - } - return "", nil -} - -//GetYamlByGit -func GetYamlByGit(url string) (string, error) { - defer common.Catchs() - var gt []models.GitPackageTable - page := 1 - size := 20 - num, err := models.GetYamlTable(>) - if err != nil { - logs.Error("Query table failed, unable to get yaml, err: ", err) - return "", err - } - logs.Info("There are: ", num, ", Table data needs to be obtained, ", gt) - compURL1 := url + "/packages" - var ch = make(chan int, len(gt)) - for i, tableValue := range gt { - go func(tv models.GitPackageTable, idx, locPage, locSize int, locUrl, locCompURL1 string) { - ok, err := GetYaml(locUrl, locCompURL1, locPage, locSize, tv, &ch) - if err == nil { - logs.Info("The current data processing is successful,i: ", idx) - } else { - logs.Error("The current data processing failed, ok: ", ok, ",i: ", idx, ", err: ", err) - } - }(tableValue, i, page, size, url, compURL1) - } - for i := 0; i < len(gt); i++ { - <-ch - } - close(ch) - return "", nil -} - -func GetYaml(url, compURL1 string, page, size int, - tableValue models.GitPackageTable, ch *chan int) (string, error) { - defer common.Catchs() - var tc GitTablePackCount - tc.TableName = tableValue.TableName - tc.Page = 0 - tc.Size = 0 - tc.Page = page - tc.Size = size - compURL2 := compURL1 + "?table_name=" + tableValue.TableName - for { - compURL := compURL2 + - "&page_num=" + strconv.Itoa(tc.Page) + "&page_size=" + strconv.Itoa(size) - body, err := util.HTTPGetCom(compURL) - var respBody map[string]interface{} - if err == nil && body != nil { - err = json.Unmarshal(body, &respBody) - if err != nil { - logs.Error(err) - return "", err - } - } else { - logs.Error("http request failed, url: ", compURL) - return "", err - } - logs.Info("start: ", tc) - if respBody["code"].(string) == "2001" { - curCount := 0 - if respBody == nil || respBody["data"] == nil || len(respBody["data"].([]interface{})) == 0 { - logs.Error("Data is empty, url: ", compURL) - return "", err - } - for i, values := range respBody["data"].([]interface{}) { - tc.Count = tc.Count + 1 - if values == nil || values == "" || len(values.(map[string]interface{})) == 0 { - curCount = curCount + 1 - continue - } - valuesX := values.(map[string]interface{}) - lock.Lock() - ok, err := ProcPackDetail(url, valuesX, tableValue, i) - lock.Unlock() - logs.Info("ok: ", ok, ", err: ", err) - curCount = curCount + 1 - } - totalPage := 0 - switch respBody["total_page"].(type) { - case string: - totalPage, _ = strconv.Atoi(respBody["total_page"].(string)) - case int: - totalPage = respBody["total_page"].(int) - case int64: - totalPage = int(respBody["total_page"].(int64)) - case float64: - totalPage = int(int64(respBody["total_page"].(float64))) - default: - totalPage = 1 - } - totalCount := 0 - switch respBody["total_count"].(type) { - case string: - totalCount, _ = strconv.Atoi(respBody["total_count"].(string)) - case int: - totalCount = respBody["total_count"].(int) - case int64: - totalCount = int(respBody["total_count"].(int64)) - case float64: - totalCount = int(int64(respBody["total_count"].(float64))) - default: - totalCount = 1 - } - if tc.Page > totalPage || tc.Size >= totalCount { - logs.Info("Processed:tableName: ", tc.TableName, "Data acquisition completed, "+ - "total pages(page_num):", totalPage, ", Total number(page_size):", totalCount, "\n", - ",current page number:", tc.Page, ",Current number: ", tc.Size, ",url: ", compURL) - break - } else { - logs.Info("current: tableName: ", tc.TableName, "Data acquisition completed, "+ - "total pages(page_num):", totalPage, ", Total number(page_size):", totalCount, "\n", - ",current page number:", tc.Page, ",Current number: ", tc.Size, ",url: ", compURL) - lock.Lock() - tc.Page = tc.Page + page - tc.Size = tc.Size + curCount - lock.Unlock() - logs.Info("After the increase: tableName: ", tc.TableName, "Data acquisition completed, "+ - "total pages(page_num):", totalPage, ", Total number(page_size):", totalCount, "\n", - ",current page number:", tc.Page, ",Current number: ", tc.Size, ",url: ", compURL) - } - logs.Info("start: ", tc) - } else { - logs.Error("Network request failed,url:", compURL) - continue - } - } - *ch <- 1 - return "", nil -} - -func ProcPackDetail(url string, value map[string]interface{}, tableValue models.GitPackageTable, i int) (string, error) { - values := value - var ge models.GitOpenEuler - GitOpenEulerData(values, &ge, tableValue) - // Query cpe data - var pc models.PackageCpe - pcError := models.GetCpePackName(&pc, ge.PackageName) - if pcError == nil && pc.Id > 0 { - ge.CpePackName = pc.CpePackName - } - logs.Info("Being processed packageName: ", ge.PackageName) - ok, _ := models.GetSingleYaml(&ge) - if ok { - logs.Info("PackageName: ", ge.PackageName, ",Already exists, no need to insert again") - gpinfo, okinfo := models.QueryCveOpeneulerDetaildataByName(ge.PackageName, ge.Version) - if !okinfo && gpinfo.DetailId == 0 { - _, pierr := GetPackageInfo(url, tableValue, ge) - if pierr != nil { - logs.Error("Failed to get package details, url: ", url, - ", PackageName: ", ge.PackageName, ",version: ", ge.Version, ",err: ", pierr) - return ge.PackageName, pierr - } - } - return ge.PackageName, nil - } - gitID, typeX, err := models.CreateYaml(&ge, tableValue) - if gitID > 0 && err == nil { - logs.Info("PackageName: ", ge.PackageName, typeX, "success, git_id: ", gitID) - } else { - logs.Error("PackageName: ", ge.PackageName, typeX, "failure, err: ", err) - if strings.ContainsAny(string(err.Error()), "Duplicate entry") { - ok, _ := models.GetSingleYaml(&ge) - if ok { - logs.Info("PackageName: ", ge.PackageName, ",Already exists, no need to insert again") - gpinfo, okinfo := models.QueryCveOpeneulerDetaildataByName(ge.PackageName, ge.Version) - if !okinfo && gpinfo.DetailId == 0 { - _, pierr := GetPackageInfo(url, tableValue, ge) - if pierr != nil { - logs.Error("Failed to get package details, url: ", url, ", PackageName: ", ge.PackageName, - ",version: ", ge.Version, ",err: ", pierr) - return ge.PackageName, pierr - } - } - return ge.PackageName, nil - } - } - return "", err - } - if typeX == "insert" && gitID > 0 { - _, pierr := GetPackageInfo(url, tableValue, ge) - if pierr != nil { - logs.Error("Failed to get package details, url: ", url, ", PackageName: ", ge.PackageName, ",version: ", ge.Version, ",err: ", pierr) - return ge.PackageName, pierr - } - } - return "", nil -} - -func GetPackageInfo(urls string, tableValue models.GitPackageTable, ge models.GitOpenEuler) (string, error) { - scpURL := urls + "/packages/packageInfo" + "?table_name=" + tableValue.TableName + "&pkg_name=" + ge.PackageName - logs.Info("PackageName: ", ge.PackageName, "Get packageInfo, request parameters: ", scpURL) - var urlS url.URL - q := urlS.Query() - q.Add("table_name", tableValue.TableName) - q.Add("pkg_name", ge.PackageName) - params := q.Encode() - req, _ := http.NewRequest("GET", urls, nil) - req.URL = &url.URL{ - Scheme: req.URL.Scheme, - Host: req.URL.Host, - RawQuery: params, - Path: "/pkgmanage/packages/packageInfo", - } - - req.URL.EscapedPath() - body, err := util.HTTPGetCom(req.URL.String()) - var respBody map[string]interface{} - if err == nil && body != nil { - err = json.Unmarshal(body, &respBody) - if err != nil { - logs.Error(err) - return "", err - } - } else { - logs.Error("http request failed, scpURL: ", scpURL) - return "", err - } - logs.Info(respBody) - if respBody["code"].(string) == "2001" { - if respBody["data"] == nil || respBody["data"] == "" { - logs.Error("Failed to get package details, scpURL: ", scpURL) - return "", errors.New("数据错误") - } - var gp models.GitPackageInfo - GitOpenEulerInfoData(respBody["data"].(map[string]interface{}), &gp, ge) - detailid, typex, err := models.CreateYamlDetail(&gp, ge) - if detailid > 0 && typex == "insert" && err == nil { - subdata := respBody["data"].(map[string]interface{}) - if subdata["subpack"] == nil { - return "", err - } - if subdata == nil || subdata["subpack"] == nil || len(subdata["subpack"].([]interface{})) == 0 { - return "", errors.New("数据错误") - } - subpack := subdata["subpack"].([]interface{}) - for _, packValuex := range subpack { - logs.Info("yaml packValuex: ", packValuex) - if packValuex == nil || packValuex == "" || len(packValuex.(map[string]interface{})) == 0 { - continue - } - packValue := packValuex.(map[string]interface{}) - var gb models.GitSubPack - gb.DetailId = gp.DetailId - switch packValue["id"].(type) { - case string: - gb.Ids, _ = strconv.ParseInt(packValue["id"].(string), 10, 64) - case int: - gb.Ids = packValue["id"].(int64) - case int64: - gb.Ids = packValue["id"].(int64) - case float64: - gb.Ids = int64(packValue["id"].(float64)) - default: - gb.Ids = 0 - } - if packValue["name"] == nil { - gb.SubPackName = "" - } else { - gb.SubPackName = packValue["name"].(string) - } - SubID, typex, err := models.CreateYamlSubPack(&gb) - if SubID > 0 && typex == "insert" && err == nil { - if packValue["provides"] != nil && len(packValue["provides"].([]interface{})) > 0 { - provides := packValue["provides"].([]interface{}) - for _, provValuex := range provides { - logs.Info("yaml provValuex: ", provValuex) - if provValuex == nil || provValuex == "" || len(provValuex.(map[string]interface{})) == 0 { - continue - } - provValue := provValuex.(map[string]interface{}) - var gs models.GitSubPackProvides - gs.SubId = SubID - switch provValue["id"].(type) { - case string: - gs.Ids, _ = strconv.ParseInt(provValue["id"].(string), 10, 64) - case int: - gs.Ids = provValue["id"].(int64) - case int64: - gs.Ids = provValue["id"].(int64) - case float64: - gs.Ids = int64(provValue["id"].(float64)) - default: - gs.Ids = 0 - } - if provValue["name"] == nil { - gs.ProvideName = "" - } else { - gs.ProvideName = provValue["name"].(string) - } - gs.Requiredby = "" - ProvideID, typexx, err := models.CreateYamlSubPackProvides(&gs) - if ProvideID > 0 && typexx == "insert" && err == nil { - if provValue["requiredby"] != nil && len(provValue["requiredby"].([]interface{})) > 0 { - requiredby := provValue["requiredby"].([]interface{}) - for _, reqValue := range requiredby { - if reqValue != nil && reqValue.(string) != "" { - var gr models.GitSubPackRequiredby - gr.ProvideId = gs.ProvideId - gr.Requiredby = reqValue.(string) - ID, typexy, err := models.CreateYamlSubPackRequiredb(&gr) - logs.Info("CreateYamlSubPackRequiredb", ID, typexy, err) - } - } - } - } - } - } - if packValue["requires"] != nil && len(packValue["requires"].([]interface{})) > 0 { - requires := packValue["requires"].([]interface{}) - for _, reqValuexx := range requires { - logs.Info("reqValuexx: ", reqValuexx) - if reqValuexx == nil || reqValuexx == "" || len(reqValuexx.(map[string]interface{})) == 0 { - continue - } - reqValuex := reqValuexx.(map[string]interface{}) - reqStr := "" - if reqValuex["providedby"] != nil && len(reqValuex["providedby"].([]interface{})) > 0 { - providedby := reqValuex["providedby"].([]interface{}) - for _, reqValue := range providedby { - if reqValue != nil && reqValue.(string) != "" { - reqStr = reqStr + reqValue.(string) + "," - } - } - } - if reqStr != "" { - reqStr = reqStr[:len(reqStr)-1] - } - var gs models.GitSubPackRequire - gs.SubId = SubID - switch reqValuex["id"].(type) { - case string: - gs.Ids, _ = strconv.ParseInt(reqValuex["id"].(string), 10, 64) - case int: - gs.Ids = reqValuex["id"].(int64) - case int64: - gs.Ids = reqValuex["id"].(int64) - case float64: - gs.Ids = int64(reqValuex["id"].(float64)) - default: - gs.Ids = 0 - } - if reqValuex["name"] == nil { - gs.RequireName = "" - } else { - gs.RequireName = reqValuex["name"].(string) - } - gs.Providedby = reqStr - RequireID, typexx, err := models.CreateYamlSubPackRequires(&gs) - logs.Info("CreateYamlSubPackRequires", RequireID, typexx, err) - } - } - } - } - } - } - return "", nil -} - func GetEulerYamlInfo(url string) (string, error) { defer common.Catchs() compUrl := url + "?community=openeuler" @@ -422,15 +32,20 @@ func GetEulerYamlInfo(url string) (string, error) { logs.Error("http Request failed, url: ", compUrl) return "", err } - if respBody == nil || respBody["data"] == nil || len(respBody["data"].([]interface{})) == 0 { + + data, ok := respBody["data"] + if !ok || len(data.([]interface{})) == 0 { logs.Error("Data is empty, url: ", compUrl) return "", fmt.Errorf("Data is empty, url: %s", compUrl) } - if respBody["code"].(float64) == 200 && respBody["total"].(float64) > 0 { + + code, ok2 := respBody["code"] + total, ok3 := respBody["total"] + if ok2 && ok3 && code.(float64) == http.StatusOK && total.(float64) > 0 { //models.TruncateOpenEulerRepoOrigin() outSyncBranch := beego.AppConfig.String("cve::out_of_sync_branch") outSyncBranchList := strings.Split(outSyncBranch, ",") - for _, values := range respBody["data"].([]interface{}) { + for _, values := range data.([]interface{}) { valuesX := values.(map[string]interface{}) ProcYamlInfo(valuesX, outSyncBranchList) } diff --git a/cve-vulner-manager/util/parsepayload.go b/cve-vulner-manager/util/parsepayload.go index 5229e477ec07e3925adcf969e5e9952432e751db..655c6d8efdd3196bc96c33217a6f1423128dedac 100644 --- a/cve-vulner-manager/util/parsepayload.go +++ b/cve-vulner-manager/util/parsepayload.go @@ -349,7 +349,9 @@ func ReadVMValue(kStr string) (value string) { mutex.Lock() defer mutex.Unlock() if _, ok := VectorMap[sKs[0]]; ok { - value = VectorMap[sKs[0]][sKs[1]] + if v, ok2 := VectorMap[sKs[0]][sKs[1]]; ok2 { + return v + } } return } @@ -370,7 +372,9 @@ func ReadVMValueV2(kStr string) (value string) { mutex.Lock() defer mutex.Unlock() if _, ok := VectorMapV2[sKs[0]]; ok { - value = VectorMapV2[sKs[0]][sKs[1]] + if v, ok2 := VectorMapV2[sKs[0]][sKs[1]]; ok2 { + return v + } } return }