From 1b15a75874637d638047a128fee6eb9c2cac6b2f Mon Sep 17 00:00:00 2001
From: yangwei999 <348134071@qq.com>
Date: Wed, 23 Apr 2025 10:35:32 +0800
Subject: [PATCH 1/3] optimize sync openeuelr packages

---
 cve-vulner-manager/controllers/hook.go        |   6 +
 .../cve-timed-task/tabletask/packages.go      |   2 +-
 cve-vulner-manager/models/ymal.go             | 301 +------------
 cve-vulner-manager/task/yaml.go               |  22 +-
 cve-vulner-manager/taskhandler/cve.go         |   1 +
 .../taskhandler/issuestatistics.go            |   2 +-
 cve-vulner-manager/taskhandler/sendemail.go   |  16 +-
 cve-vulner-manager/taskhandler/yaml.go        | 410 +++++++-----------
 8 files changed, 199 insertions(+), 561 deletions(-)

diff --git a/cve-vulner-manager/controllers/hook.go b/cve-vulner-manager/controllers/hook.go
index 47c266e..9f9e801 100644
--- a/cve-vulner-manager/controllers/hook.go
+++ b/cve-vulner-manager/controllers/hook.go
@@ -96,6 +96,8 @@ const (
 	cmdCheckIssue = "/check-issue"
 	reasonCommand = "/reason"
 	reasonX       = "xxxxxx"
+
+	openEulerBotName = "openeuler-ci-bot"
 )
 
 var comLock sync.Mutex
@@ -232,6 +234,10 @@ func (c *HookEventControllers) handleIssue() {
 		}
 	}
 	if issueHook.Action == "open" {
+		if issueHook.User.Login == openEulerBotName {
+			return
+		}
+
 		owner, token := common.GetOwnerAndToken("", organizationID)
 		issueTmp := models.IssueTemplate{IssueNum: issueHook.Iid, IssueId: issueHook.Issue.Id}
 		err = models.GetIssueTemplateByColName(&issueTmp, "issue_num", "issue_id")
diff --git a/cve-vulner-manager/cve-timed-task/tabletask/packages.go b/cve-vulner-manager/cve-timed-task/tabletask/packages.go
index f4f43eb..e168d27 100644
--- a/cve-vulner-manager/cve-timed-task/tabletask/packages.go
+++ b/cve-vulner-manager/cve-timed-task/tabletask/packages.go
@@ -101,7 +101,7 @@ type response struct {
 }
 
 func getPackageFromNG(page int) ([]Packages, error) {
-	url := fmt.Sprintf("%s/api/v1/packages?page=%d&per_page=%d",
+	url := fmt.Sprintf("%s/api/v1/packages?page=%d&per_page=%d&exclude_org=ascend",
 		beego.AppConfig.String("cve-manager-ng::host"), page, pageSize)
 
 	const maxRetry = 3
diff --git a/cve-vulner-manager/models/ymal.go b/cve-vulner-manager/models/ymal.go
index 1b9bca1..cd1a4ce 100644
--- a/cve-vulner-manager/models/ymal.go
+++ b/cve-vulner-manager/models/ymal.go
@@ -5,277 +5,6 @@ import (
 	"github.com/astaxie/beego/orm"
 )
 
-func GetYamlTable(gt *[]GitPackageTable) (int64, error) {
-	o := orm.NewOrm()
-	num, errx := o.Raw("select *" +
-		" from cve_git_package_table").QueryRows(gt)
-	if errx != nil {
-		logs.Error("GetYamlTable, Get data does not exist, err: ", errx)
-	}
-	return num, errx
-}
-
-func GetCpePackName(pc *PackageCpe, packName string) (error) {
-	o := orm.NewOrm()
-	errx := o.Raw("select id, cpe_packname"+
-		" from cve_package_cpe where packname = ?", packName).QueryRow(pc)
-	return errx
-}
-
-func CreateYamlTable(gt *GitPackageTable) (TableId int64, err error) {
-	o := orm.NewOrm()
-	var localgt GitPackageTable
-	if gt.TableName == "openEuler_LTS_20.03" {
-		gt.Brands = "openEuler-20.03-LTS"
-	} else {
-		gt.Brands = "master"
-	}
-	errx := o.Raw("select *"+
-		" from cve_git_package_table where table_name = ? ", gt.TableName).QueryRow(&localgt)
-	if errx != nil {
-		var TableId int64
-		if TableId, err = o.Insert(gt); err == nil {
-			logs.Info("CreateYamlTable, insert cve_git_package_table success, "+
-				"TableId: ", TableId, ", tableName: ", gt.TableName)
-		} else {
-			logs.Error("CreateYamlTable, insert cve_git_package_table failed,"+
-				" tableName:", gt.TableName, ", err: ", err)
-			return 0, err
-		}
-		return TableId, nil
-	} else {
-		gt.TableId = localgt.TableId
-		gt.Brands = localgt.Brands
-		if num, err := o.Update(gt); err == nil {
-			logs.Info("CreateYamlTable, update cve_git_package_table success, "+
-				"num: ", num, ", TableName: ", gt.TableName)
-		} else {
-			logs.Error("CreateYamlTable, update cve_git_package_table failed, "+
-				"TableName:", gt.TableName, ", err: ", err)
-			return 0, err
-		}
-		return gt.TableId, nil
-	}
-}
-
-func GetSingleYaml(ge *GitOpenEuler) (bool, error) {
-	o := orm.NewOrm()
-	if ge.Version != "" && ge.Release != "" {
-		localge := GitOpenEuler{PackageName: ge.PackageName, Version: ge.Version, Status: 1}
-		err := o.Read(&localge, "PackageName", "Version", "Release")
-		logs.Info("GetSingleYaml1, Query yaml data results: ", ge,
-			", localge: ", localge, ", err: ", err)
-		if err == orm.ErrNoRows {
-			return false, err
-		} else if err == orm.ErrMissPK {
-			return false, err
-		} else {
-			return true, err
-		}
-	} else {
-		localge := GitOpenEuler{PackageName: ge.PackageName, Status: 1}
-		err := o.Read(&localge, "PackageName")
-		logs.Info("GetSingleYaml2, Query yaml data results: ", ge,
-			", localge: ", localge, ", err: ", err)
-		if err == orm.ErrNoRows {
-			return false, err
-		} else if err == orm.ErrMissPK {
-			return false, err
-		} else {
-			return true, err
-		}
-	}
-}
-
-func InsertOpenEulerRelate(getr *GitOpenEulerTableRelate) (id int64, typex string, err error) {
-	o := orm.NewOrm()
-	var localge GitOpenEulerTableRelate
-	errx := o.Raw("select *"+
-		" from cve_git_open_euler_table_relate where git_id = ? and table_name = ?",
-		getr.GitId, getr.TableName).QueryRow(&localge)
-	if errx != nil {
-		var relateId int64
-		if relateId, err = o.Insert(getr); err == nil {
-			logs.Info("InsertOpenEulerRelate, insert cve_git_open_euler_table_relate success, "+
-				"RelateId: ", relateId, "tableName: ", getr.TableName)
-		} else {
-			logs.Error("InsertOpenEulerRelate, insert cve_git_open_euler_table_relate failed, "+
-				"tableName: ", getr.TableName, ", err: ", err)
-			return 0, "insert", err
-		}
-		return relateId, "insert", nil
-	} else {
-		getr.RelateId = localge.RelateId
-		if _, err := o.Update(getr); err == nil {
-			logs.Info("InsertOpenEulerRelate, update cve_git_open_euler_table_relate success, "+
-				"RelateId: ", getr.RelateId, "tableName: ", getr.TableName)
-		} else {
-			logs.Error("InsertOpenEulerRelate, update cve_git_open_euler_table_relate failed, "+
-				"tableName: ", getr.TableName, ", err: ", err)
-			return getr.RelateId, "update", err
-		}
-		return getr.RelateId, "update", nil
-	}
-}
-
-func CreateYaml(ge *GitOpenEuler, tableValue GitPackageTable) (id int64, typex string, err error) {
-	o := orm.NewOrm()
-	var localge GitOpenEuler
-	var getr GitOpenEulerTableRelate
-	getr.TableName = tableValue.TableName
-	getr.Brands = tableValue.Brands
-	errx := o.Raw("select *"+
-		" from cve_git_open_euler where package_name = ? and version = ?",
-		ge.PackageName, ge.Version).QueryRow(&localge)
-	if errx != nil {
-		logs.Info("createYaml: ", localge, ",errx: ", errx)
-		ge.Status = 1
-		var GitId int64
-		if GitId, err = o.Insert(ge); err == nil {
-			logs.Info("CreateYaml, insert cve_git_open_euler success, GitId: ",
-				GitId, ", packName: ", ge.PackageName)
-		} else {
-			logs.Error("CreateYaml, insert cve_git_open_euler failed, "+
-				"packName: ", ge.PackageName, ", err: ", err)
-			return 0, "insert", err
-		}
-		getr.GitId = GitId
-		InsertOpenEulerRelate(&getr)
-		return GitId, "insert", nil
-	} else {
-		logs.Info("createYaml: ", localge, "GitOpenEuler: ", ge)
-		ge.Status = 1
-		ge.GitId = localge.GitId
-		if _, err := o.Update(ge); err == nil {
-			logs.Info("CreateYaml, update cve_git_open_euler success, "+
-				"GitId: ", ge.GitId, ", packName: ", ge.PackageName)
-		} else {
-			logs.Error("CreateYaml, update cve_git_open_euler failed, "+
-				"packName: ", ge.PackageName, ", err: ", err)
-			return ge.GitId, "update", err
-		}
-		getr.GitId = ge.GitId
-		InsertOpenEulerRelate(&getr)
-		return ge.GitId, "update", nil
-	}
-}
-
-func CreateYamlDetail(gp *GitPackageInfo, ge GitOpenEuler) (id int64, typex string, err error) {
-	o := orm.NewOrm()
-	var localgp GitPackageInfo
-	errx := o.Raw("select *"+
-		" from cve_git_package_info where git_id = ? and package_name = ? and version = ?",
-		ge.GitId, gp.PackageName, gp.Version).QueryRow(&localgp)
-	if errx != nil {
-		gp.Status = 0
-		var DetailId int64
-		if DetailId, err = o.Insert(gp); err == nil {
-			logs.Info("CreateYamlDetail, insert cve_git_package_info success, "+
-				"DetailId: ", DetailId, ",PackageName: ", gp.PackageName)
-		} else {
-			logs.Error("CreateYamlDetail, insert cve_git_package_info failed, "+
-				"PackageName: ", gp.PackageName, ",err: ", err)
-			return 0, "insert", err
-		}
-		return DetailId, "insert", nil
-	} else {
-		gp.DetailId = localgp.DetailId
-		if _, err := o.Update(gp); err == nil {
-			logs.Info("CreateYamlDetail, update cve_git_package_info success, "+
-				"DetailId: ", gp.DetailId, ",packName: ", gp.PackageName)
-		} else {
-			logs.Error("CreateYamlDetail, update cve_git_package_info failed, "+
-				"packName: ", gp.PackageName, ",err: ", err)
-			return gp.DetailId, "update", err
-		}
-		return gp.DetailId, "update", nil
-	}
-}
-
-func CreateYamlSubPack(gb *GitSubPack) (SubId int64, typex string, err error) {
-	o := orm.NewOrm()
-	var localgb GitSubPack
-	errx := o.Raw("select *"+
-		" from cve_git_sub_pack where detail_id = ? and ids = ? and sub_pack_name = ?",
-		gb.DetailId, gb.Ids, gb.SubPackName).QueryRow(&localgb)
-	if errx != nil {
-		var SubId int64
-		if SubId, err = o.Insert(gb); err == nil {
-			logs.Info("CreateYamlSubPack, insert cve_git_sub_pack success,"+
-				" SubId: ", SubId, ",SubPackName: ", gb.SubPackName)
-		} else {
-			logs.Error("CreateYamlSubPack, insert cve_git_sub_pack failed, "+
-				"SubPackName: ", gb.SubPackName, ",err: ", err)
-			return 0, "insert", err
-		}
-		return SubId, "insert", nil
-	}
-	return localgb.SubId, "update", errx
-}
-
-func CreateYamlSubPackProvides(gs *GitSubPackProvides) (ProvideId int64, typex string, err error) {
-	o := orm.NewOrm()
-	var localgs GitSubPackProvides
-	errx := o.Raw("select *"+
-		" from cve_git_sub_pack_provides where sub_id = ? and ids = ? and provide_name = ?",
-		gs.SubId, gs.Ids, gs.ProvideName).QueryRow(&localgs)
-	if errx != nil {
-		var ProvideId int64
-		if ProvideId, err = o.Insert(gs); err == nil {
-			logs.Info("CreateYamlSubPack, insert cve_git_sub_pack_provides success, "+
-				"ProvideId: ", ProvideId, ", ProvideName: ", gs.ProvideName)
-		} else {
-			logs.Error("CreateYamlSubPack, insert cve_git_sub_pack_provides failed, "+
-				"ProvideName: ", gs.ProvideName, ",err: ", err)
-			return 0, "insert", err
-		}
-		return ProvideId, "insert", nil
-	}
-	return localgs.ProvideId, "update", errx
-}
-
-func CreateYamlSubPackRequiredb(gs *GitSubPackRequiredby) (Id int64, typex string, err error) {
-	o := orm.NewOrm()
-	var localgs GitSubPackRequiredby
-	errx := o.Raw("select *"+
-		" from cve_git_sub_pack_requiredby where provide_id = ? and requiredb = ?",
-		gs.ProvideId, gs.Requiredby).QueryRow(&localgs)
-	if errx != nil {
-		var Id int64
-		if Id, err = o.Insert(gs); err == nil {
-			logs.Info("CreateYamlSubPackRequiredb, insert cve_git_sub_pack_requiredby success, "+
-				"ID: ", Id, ", requiredb: ", gs.Requiredby)
-		} else {
-			logs.Error("CreateYamlSubPackRequiredb, insert cve_git_sub_pack_requiredby failed, "+
-				"requiredb: ", gs.Requiredby, ", err: ", err)
-			return 0, "insert", err
-		}
-		return Id, "insert", nil
-	}
-	return localgs.Id, "update", errx
-}
-
-func CreateYamlSubPackRequires(gs *GitSubPackRequire) (RequireId int64, typex string, err error) {
-	o := orm.NewOrm()
-	var localgs GitSubPackRequire
-	errx := o.Raw("select *"+
-		" from cve_git_sub_pack_require where sub_id = ? and ids = ? and require_name = ?",
-		gs.SubId, gs.Ids, gs.RequireName).QueryRow(&localgs)
-	if errx != nil {
-		var RequireId int64
-		if RequireId, err = o.Insert(gs); err == nil {
-			logs.Info("CreateYamlSubPackRequires, insert cve_git_sub_pack_require success,"+
-				" RequireId: ", RequireId, ",RequireName: ", gs.RequireName)
-		} else {
-			logs.Error("CreateYamlSubPackRequires, insert cve_git_sub_pack_require failed, "+
-				"RequireName: ", gs.RequireName, ", err: ", err)
-			return 0, "insert", err
-		}
-		return RequireId, "insert", nil
-	}
-	return localgs.RequireId, "update", errx
-}
-
 func QueryEulerRepoOrigin(ogk *OpenEulerRepoOrigin, field ...string) error {
 	o := orm.NewOrm()
 	err := o.Read(ogk, field...)
@@ -305,6 +34,24 @@ func UpdateEulerRepoOrigin(ogk *OpenEulerRepoOrigin, fields ...string) error {
 	return err
 }
 
+func DeleteEulerRepoOrigin() error {
+	o := orm.NewOrm()
+	_, err := o.Raw("delete from cve_open_euler_repo_origin").Exec()
+	return err
+}
+
+func DeleteGitOpenEuler() error {
+	o := orm.NewOrm()
+	_, err := o.Raw("delete from cve_git_open_euler where source != ? ", "ng").Exec()
+	return err
+}
+
+func DeleteGitPackageInfo() error {
+	o := orm.NewOrm()
+	_, err := o.Raw("delete from cve_git_package_info").Exec()
+	return err
+}
+
 func GetEulerYamlList(yamlId int64, count int) (oer []OpenEulerRepoOrigin) {
 	o := orm.NewOrm()
 	num, geErr := o.Raw("select *"+
@@ -325,15 +72,3 @@ func GetGitOpenEulerList(gitId int64, count int) (goe []GitOpenEuler) {
 	}
 	return
 }
-
-func DelOpenEulerRepoOriginByName(packageName string) {
-	o := orm.NewOrm()
-	_ = o.Raw("DELETE FROM cve_open_euler_repo_origin WHERE package_name = ?", packageName).QueryRow()
-	return
-}
-
-func TruncateOpenEulerRepoOrigin() {
-	o := orm.NewOrm()
-	_ = o.Raw("TRUNCATE table cve_open_euler_repo_origin").QueryRow()
-	return
-}
diff --git a/cve-vulner-manager/task/yaml.go b/cve-vulner-manager/task/yaml.go
index fd9261c..540290f 100644
--- a/cve-vulner-manager/task/yaml.go
+++ b/cve-vulner-manager/task/yaml.go
@@ -1,30 +1,10 @@
 package task
 
 import (
-	"errors"
-
-	"github.com/astaxie/beego"
-	"github.com/astaxie/beego/logs"
-
-	"cvevulner/common"
 	"cvevulner/taskhandler"
 )
 
 // GetEulerYamlData Get yaml data source
 func GetEulerYamlData() error {
-	defer common.Catchs()
-	logs.Info("Get the euleryaml data source task started")
-	// Query the cve to be processed, 1: add; 2: modify
-	eulerUrl := beego.AppConfig.String("yaml::eulerurl")
-	if eulerUrl == "" {
-		logs.Error("GetEulerYamlData, config yaml::eulerUrl, error: invalid value")
-		return errors.New("eulerUrl value is nil")
-	}
-	_, err := taskhandler.GetEulerYamlInfo(eulerUrl)
-	// Synchronize other sources of yaml version
-	taskhandler.SyncEulerYaml()
-	// Delete historical yaml source version data that does not exist
-	taskhandler.DelHistoryEulerYaml()
-	logs.Info("End of the task of obtaining euleryaml data source")
-	return err
+	return taskhandler.SyncEulerPackages()
 }
diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go
index 8136b22..7494768 100644
--- a/cve-vulner-manager/taskhandler/cve.go
+++ b/cve-vulner-manager/taskhandler/cve.go
@@ -29,6 +29,7 @@ var (
 	pkgLock sync.Mutex
 	lockx   sync.Mutex
 	dbLock  sync.Mutex
+	lock    sync.Mutex
 )
 var ewg sync.WaitGroup
 
diff --git a/cve-vulner-manager/taskhandler/issuestatistics.go b/cve-vulner-manager/taskhandler/issuestatistics.go
index 91868d9..dd5d38e 100644
--- a/cve-vulner-manager/taskhandler/issuestatistics.go
+++ b/cve-vulner-manager/taskhandler/issuestatistics.go
@@ -245,7 +245,7 @@ func IssueStatistics(beforeMonth, prcnum int, owner string) error {
 	zipFileName := excelPath
 	if is.cveCount > 0 {
 		if templateId > 0 {
-			cBody := fmt.Sprintf("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, 10) + "个, 其中" +
+			cBody := fmt.Sprint("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, 10) + "个, 其中" +
 				strconv.FormatInt(is.HighCveCount, 10) + "个7分以上漏洞, 详情见附件, 7分以上CVE请在周三之前解决. \r\n" +
 				"已经分析完毕的issue请maintainer尽快关掉, 否则影响数据统计; 提交PR时要关联issue, 若CVE在之前PR解决, 请PR提交人编辑PR信息将issue关联上. \r\n")
 			sendError := SendEmail(zipFileName, 2, cBody, "")
diff --git a/cve-vulner-manager/taskhandler/sendemail.go b/cve-vulner-manager/taskhandler/sendemail.go
index 2e3f330..9981807 100644
--- a/cve-vulner-manager/taskhandler/sendemail.go
+++ b/cve-vulner-manager/taskhandler/sendemail.go
@@ -3,13 +3,8 @@ package taskhandler
 import (
 	"archive/zip"
 	"bytes"
-	"cvevulner/common"
-	"cvevulner/models"
 	"encoding/base64"
 	"fmt"
-	"github.com/astaxie/beego"
-	"github.com/astaxie/beego/logs"
-	"gopkg.in/gomail.v2"
 	"io"
 	"io/ioutil"
 	"mime"
@@ -19,6 +14,13 @@ import (
 	"strconv"
 	"strings"
 	"time"
+
+	"github.com/astaxie/beego"
+	"github.com/astaxie/beego/logs"
+	"gopkg.in/gomail.v2"
+
+	"cvevulner/common"
+	"cvevulner/models"
 )
 
 // define email interface, and implemented auth and send method
@@ -137,7 +139,7 @@ func SendEmail(attchStr string, flag int, cBody, subject string) error {
 		//}
 		//
 		subject := "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime()
-		body := fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been " +
+		body := fmt.Sprint("hi all: \r\n The list of cvrf format files that have been " +
 			"uploaded to the file server is as follows: \r\n" + strings.ReplaceAll(attchStr, "download/", ""))
 		emailCc := []string{}
 		sendErr := SendCommonMail(toEmailName, emailCc, subject, body, attchStr)
@@ -368,7 +370,7 @@ func SendCvrfEmail(subFileSlice []string) {
 	}
 	subject := "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime()
 	fileStr := strings.Join(subFileSlice, "\r\n")
-	body := fmt.Sprintf("hi all: \r\n The list of cvrf format files that have been uploaded to the file server is as follows: \r\n" + fileStr)
+	body := fmt.Sprint("hi all: \r\n The list of cvrf format files that have been uploaded to the file server is as follows: \r\n" + fileStr)
 	sendErr := SendCommonMail(toEmailName, []string{}, subject, body, "")
 	if sendErr == nil {
 		logs.Info("Notify cvrf that the email was sent successfully!")
diff --git a/cve-vulner-manager/taskhandler/yaml.go b/cve-vulner-manager/taskhandler/yaml.go
index ad46f48..39d8aa4 100644
--- a/cve-vulner-manager/taskhandler/yaml.go
+++ b/cve-vulner-manager/taskhandler/yaml.go
@@ -1,290 +1,204 @@
 package taskhandler
 
 import (
-	"encoding/json"
 	"fmt"
 	"net/http"
 	"strings"
-	"sync"
-
-	"cvevulner/common"
-	"cvevulner/models"
-	"cvevulner/util"
 
 	"github.com/astaxie/beego"
 	"github.com/astaxie/beego/logs"
+	"github.com/opensourceways/server-common-lib/utils"
+
+	"cvevulner/common"
+	"cvevulner/models"
 )
 
-var lock sync.Mutex
+const (
+	packageUrl = "https://magicapi.osinfra.cn/repo/versions"
+	pageSize   = 1000
+)
 
-func GetEulerYamlInfo(url string) (string, error) {
-	defer common.Catchs()
-	compUrl := url + "?community=openeuler"
-	body, err := util.HTTPGetCom(compUrl)
-	var respBody map[string]interface{}
-	if err == nil && body != nil {
-		err = json.Unmarshal(body, &respBody)
-		if err != nil {
-			logs.Error(err)
-			return "", err
-		}
-	} else {
-		logs.Error("http Request failed， url: ", compUrl)
-		return "", err
+func SyncEulerPackages() error {
+	data, err := getEulerPackages()
+	if err != nil {
+		return err
 	}
 
-	data, ok := respBody["data"]
-	if !ok || len(data.([]interface{})) == 0 {
-		logs.Error("Data is empty, url: ", compUrl)
-		return "", fmt.Errorf("Data is empty, url: %s", compUrl)
+	if err = cleanOldData(); err != nil {
+		return err
 	}
 
-	code, ok2 := respBody["code"]
-	total, ok3 := respBody["total"]
-	if ok2 && ok3 && code.(float64) == http.StatusOK && total.(float64) > 0 {
-		//models.TruncateOpenEulerRepoOrigin()
-		outSyncBranch := beego.AppConfig.String("cve::out_of_sync_branch")
-		outSyncBranchList := strings.Split(outSyncBranch, ",")
-		for _, values := range data.([]interface{}) {
-			valuesX := values.(map[string]interface{})
-			ProcYamlInfo(valuesX, outSyncBranchList)
-		}
-	}
-	return "", nil
-}
+	for _, v := range data {
+		for k, detail := range v {
+			originByVersion := groupByVersion(k, detail)
+			for _, origin := range originByVersion {
+				if _, err = models.InsertEulerRepoOrigin(&origin); err != nil {
+					logs.Error("insert euler repo origin error:", err.Error())
+					continue
+				}
+
+				goeId, err := models.InsertYamlData(&models.GitOpenEuler{
+					PackageName: origin.PackageName,
+					Version:     origin.Version,
+					PackageId:   0,
+					TableId:     0,
+					Status:      1,
+					Source:      "openeuler",
+				})
+				if err != nil {
+					logs.Error("insert git_open_euler error:", err.Error())
+					continue
+				}
+
+				gpi := models.GitPackageInfo{
+					GitId:       goeId,
+					PackageName: origin.PackageName,
+					Version:     origin.Version,
+					Summary:     origin.Summary,
+					Decription:  origin.Decription,
+					Status:      0,
+				}
 
-// Parse the received json data
-func ProcYamlInfo(valuesX map[string]interface{}, outSyncBranch []string) {
-	if valuesX != nil && len(valuesX) > 0 {
-		for repKey, repValue := range valuesX {
-			if repKey != "" && len(repKey) > 1 {
-				var branchs = make(map[string]string)
-				//Delete all data first, then insert new data
-				//models.DelOpenEulerRepoOriginByName(repKey)
-				value := repValue.(map[string]interface{})
-				brValue, brOk := value["branch_detail"]
-				if brOk {
-					brKey := brValue.([]interface{})
-					if brKey != nil && len(brKey) > 0 {
-						for _, branchValue := range brKey {
-							var er models.OpenEulerRepoOrigin
-							er.PackageName = repKey
-							v := branchValue.(map[string]interface{})
-							var ver string
-							for k, vv := range v {
-								if strings.EqualFold(k, "version") {
-									ver = vv.(string)
-									if _, ok := branchs[ver]; !ok {
-										branchs[ver] = ""
-									}
-								}
-							}
-							for vKey, vu := range v {
-								switch vKey {
-								case "summary":
-									er.Summary = vu.(string)
-								case "version":
-									er.Version = vu.(string)
-								case "brname":
-									er.Branchs = vu.(string)
-									if !filterDataInSlice(er.Branchs, outSyncBranch) {
-										branch, ok := branchs[ver]
-										if !ok {
-											branchs[ver] = vu.(string)
-										} else {
-											if len(branch) == 0 {
-												branchs[ver] = vu.(string)
-											} else {
-												branchs[ver] = branch + "," + vu.(string)
-											}
-										}
-									}
-								case "description":
-									switch vu.(type) {
-									case []interface{}:
-										des := vu.([]interface{})
-										for _, d := range des {
-											if er.Decription == "" {
-												er.Decription = d.(string)
-											} else {
-												er.Decription = er.Decription + "," + d.(string)
-											}
-										}
-									case string:
-										er.Decription = vu.(string)
-									}
-								}
-							}
-							if len(er.Branchs) > 0 {
-								outFlag := false
-								for _, outBranch := range outSyncBranch {
-									if outBranch == er.Branchs {
-										outFlag = true
-										break
-									}
-								}
-								if outFlag {
-									continue
-								}
-							}
-							UpdateYamlToDb(er, branchs)
-						}
-					}
+				if _, err = models.InsertGitPackageInfo(&gpi); err != nil {
+					logs.Error("insert git_package_info error:", err.Error())
 				}
 			}
 		}
 	}
-	return
+
+	return nil
 }
 
-// Update data to the database
-func UpdateYamlToDb(er models.OpenEulerRepoOrigin, branchs map[string]string) {
-	if len(er.PackageName) > 0 && len(er.Version) > 0 && !strings.Contains(er.Version, "%") {
-		var oro models.OpenEulerRepoOrigin
-		oro.PackageName = er.PackageName
-		oro.Version = er.Version
-		branch := branchs[er.Version]
-		repErr := models.QueryEulerRepoOrigin(&oro, "PackageName", "Version")
-		if oro.Id > 0 {
-			if len(oro.Decription) < 2 && len(er.Decription) > 2 {
-				oro.Decription = er.Decription
-			}
-			oro.Branchs = branch
-			if len(oro.Summary) < 2 && len(er.Summary) > 2 {
-				oro.Summary = er.Summary
-			}
-			oro.UpdateTime = common.GetCurTime()
-			oro.Status = 1
-			oro.Repo = er.PackageName
-			upErr := models.UpdateEulerRepoOrigin(&oro, "Decription", "Branchs", "Summary", "UpdateTime", "Repo")
-			logs.Info("UpdateEulerRepoOrigin, upErr: ", upErr)
-		} else {
-			logs.Info(repErr)
-			oro.Decription = er.Decription
-			oro.Branchs = branch
-			oro.Summary = er.Summary
-			oro.UpdateTime = common.GetCurTime()
-			oro.PackageName = er.PackageName
-			oro.Version = er.Version
-			oro.Status = 1
-			oro.Repo = er.PackageName
-			oro.CreateTime = common.GetCurTime()
-			id, inErr := models.InsertEulerRepoOrigin(&oro)
-			logs.Info("InsertEulerRepoOrigin, id, inErr: ", id, inErr)
+type BranchDetail struct {
+	Summary     string      `json:"summary"`
+	PackageName string      `json:"package_name"`
+	Description interface{} `json:"description"`
+	Version     string      `json:"version"`
+	BRName      string      `json:"brname"`
+}
+
+func (d BranchDetail) IsValid() bool {
+	outSyncBranch := beego.AppConfig.String("cve::out_of_sync_branch")
+	outSyncBranchList := strings.Split(outSyncBranch, ",")
+	for _, v := range outSyncBranchList {
+		if v == d.BRName {
+			return false
 		}
-	} else {
-		logs.Error("UpdateYamlToDb, The version information of the current warehouse is wrong, er: ", er)
 	}
+
+	if d.PackageName != "" && d.Version != "" && !strings.Contains(d.Version, "%") {
+		return true
+	}
+
+	return false
 }
 
-// Synchronize other sources of yaml version
-func SyncEulerYaml() {
-	// Query raw data
-	yamlId := int64(0)
-	for {
-		yamlList := models.GetEulerYamlList(yamlId, 100)
-		if len(yamlList) > 0 {
-			for _, yl := range yamlList {
-				if len(yl.PackageName) < 2 || len(yl.Version) < 1 {
-					logs.Error("yl: ", yl)
-					continue
-				}
-				yamlId = yl.Id
-				var ge models.GitOpenEuler
-				ge.PackageName = yl.PackageName
-				ge.Version = yl.Version
-				geErr := models.QueryYamlData(&ge, "PackageName", "Version")
-				if ge.GitId == 0 {
-					logs.Info("yaml data does not exist, need to insert data, err: ",
-						geErr, ",PackageName: ", yl.PackageName, ", Version: ", yl.Version)
-					AddYamlData(&ge, yl)
-				} else {
-					EditYamlData(&ge, yl)
-				}
-			}
-		} else {
-			break
+func (d BranchDetail) GenerateDescription() string {
+	switch d.Description.(type) {
+	case []interface{}:
+		var t []string
+		for _, v := range d.Description.([]interface{}) {
+			t = append(t, v.(string))
 		}
+
+		return strings.Join(t, ",")
+	case string:
+		return d.Description.(string)
+	default:
+		return ""
 	}
 }
 
-// Delete historical yaml source version data that does not exist
-func DelHistoryEulerYaml() {
-	gitId := int64(0)
-	for {
-		yamlList := models.GetGitOpenEulerList(gitId, 100)
-		if len(yamlList) > 0 {
-			for _, yl := range yamlList {
-				if len(yl.PackageName) < 2 || len(yl.Version) < 1 {
-					logs.Error("yl: ", yl)
-					continue
-				}
-				gitId = yl.GitId
-				var oro models.OpenEulerRepoOrigin
-				oro.PackageName = yl.PackageName
-				oro.Version = yl.Version
-				repErr := models.QueryEulerRepoOrigin(&oro, "PackageName", "Version")
-				if oro.Id == 0 {
-					logs.Info("yaml data does not exist, delete data, err: ",
-						repErr, ",PackageName: ", yl.PackageName, ", Version: ", yl.Version)
-					yl.Status = 2
-					upErr := models.UpdateYamlData(&yl, "Status")
-					if upErr == nil {
-						ge := models.GitPackageInfo{PackageName: yl.PackageName, Version: yl.Version}
-						qErr := models.QueryGitPackageInfo(&ge, "PackageName", "Version")
-						if qErr == nil {
-							ge.Status = 2
-							upgErr := models.UpdateGitPackageInfo(&ge, "Status")
-							if upgErr != nil {
-								logs.Error("UpdateGitPackageInfo, upgErr: ", upgErr)
-							}
-						}
-					}
-				}
+func cleanOldData() error {
+	if err := models.DeleteEulerRepoOrigin(); err != nil {
+		return err
+	}
+
+	if err := models.DeleteGitOpenEuler(); err != nil {
+		return err
+	}
+
+	if err := models.DeleteGitPackageInfo(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func groupByVersion(path string, detail []BranchDetail) map[string]models.OpenEulerRepoOrigin {
+	split := strings.Split(path, "/")
+	if len(split) < splitLen {
+		logs.Error("path error", path)
+
+		return nil
+	}
+
+	originByVersion := make(map[string]models.OpenEulerRepoOrigin)
+	for _, d := range detail {
+		if !d.IsValid() {
+			continue
+		}
+
+		if ov, ok := originByVersion[d.Version]; ok {
+			ov.Branchs += fmt.Sprintf(",%s", d.BRName)
+
+			if ov.Summary == "" {
+				ov.Summary = d.Summary
 			}
+
+			if ov.Decription == "" {
+				ov.Decription = d.GenerateDescription()
+			}
+
+			originByVersion[d.Version] = ov
 		} else {
-			break
+			originByVersion[d.Version] = models.OpenEulerRepoOrigin{
+				PackageName: d.PackageName,
+				Version:     d.Version,
+				Summary:     d.Summary,
+				Decription:  d.GenerateDescription(),
+				Branchs:     d.BRName,
+				Status:      1,
+				CreateTime:  common.GetCurTime(),
+				UpdateTime:  common.GetCurTime(),
+				Repo:        split[1],
+			}
 		}
 	}
-}
 
-func AddYamlData(ge *models.GitOpenEuler, yl models.OpenEulerRepoOrigin) {
-	ge.PackageName = yl.PackageName
-	ge.Version = yl.Version
-	ge.PackageId = 0
-	ge.TableId = 0
-	//ge.TableName = yl.Branchs
-	ge.Status = 1
-	ymalId, yErr := models.InsertYamlData(ge)
-	if yErr == nil {
-		ProcGitPackageInfo(ymalId, yl)
-	}
+	return originByVersion
 }
 
-func EditYamlData(ge *models.GitOpenEuler, yl models.OpenEulerRepoOrigin) {
-	ge.Status = 1
-	yErr := models.UpdateYamlData(ge, "Status")
-	logs.Info("UpdateYamlData, yErr: ", yErr)
-	ProcGitPackageInfo(ge.GitId, yl)
+type Response struct {
+	Code    int                         `json:"code"`
+	Message string                      `json:"message"`
+	Data    []map[string][]BranchDetail `json:"data"`
 }
 
-func ProcGitPackageInfo(ymalId int64, yl models.OpenEulerRepoOrigin) {
-	gp := models.GitPackageInfo{GitId: ymalId, PackageName: yl.PackageName, Version: yl.Version}
-	gpErr := models.QueryGitPackageInfo(&gp, "GitId", "PackageName", "Version")
-	if gpErr != nil {
-		gp.GitId = ymalId
-		gp.PackageName = yl.PackageName
-		gp.Version = yl.Version
-		gp.Summary = yl.Summary
-		gp.Decription = yl.Decription
-		gp.Status = 0
-		id, iErr := models.InsertGitPackageInfo(&gp)
-		logs.Info("InsertGitPackageInfo, id, iErr", id, iErr)
-	} else {
-		gp.Summary = yl.Summary
-		gp.Decription = yl.Decription
-		gp.Status = 0
-		uErr := models.UpdateGitPackageInfo(&gp, "Summary", "Decription", "Status")
-		logs.Info("UpdateGitPackageInfo, uErr: ", uErr)
+func getEulerPackages() ([]map[string][]BranchDetail, error) {
+	cli := utils.NewHttpClient(3)
+	var result []map[string][]BranchDetail
+	page := 1
+	for {
+		url := fmt.Sprintf("%s?community=openeuler&pageSize=%d&page=%d", packageUrl, pageSize, page)
+		req, err := http.NewRequest(http.MethodGet, url, nil)
+		if err != nil {
+			return nil, err
+		}
+
+		var resp Response
+		if _, err = cli.ForwardTo(req, &resp); err != nil {
+			return nil, err
+		}
+
+		result = append(result, resp.Data...)
+
+		if len(resp.Data) < pageSize {
+			break
+		}
+
+		page++
 	}
+
+	return result, nil
 }
-- 
Gitee


From f3628cbd29dc66fa66649ecc9578491e18a2a8a3 Mon Sep 17 00:00:00 2001
From: yangwei999 <348134071@qq.com>
Date: Tue, 6 May 2025 16:50:42 +0800
Subject: [PATCH 2/3] fix ci

---
 cve-vulner-manager/models/ymal.go             |  3 +
 .../taskhandler/issuestatistics.go            |  4 +-
 cve-vulner-manager/taskhandler/sendemail.go   |  3 +-
 cve-vulner-manager/taskhandler/yaml.go        | 89 +++++++++++--------
 4 files changed, 58 insertions(+), 41 deletions(-)

diff --git a/cve-vulner-manager/models/ymal.go b/cve-vulner-manager/models/ymal.go
index cd1a4ce..5bb3e3c 100644
--- a/cve-vulner-manager/models/ymal.go
+++ b/cve-vulner-manager/models/ymal.go
@@ -34,18 +34,21 @@ func UpdateEulerRepoOrigin(ogk *OpenEulerRepoOrigin, fields ...string) error {
 	return err
 }
 
+// DeleteEulerRepoOrigin delete all data
 func DeleteEulerRepoOrigin() error {
 	o := orm.NewOrm()
 	_, err := o.Raw("delete from cve_open_euler_repo_origin").Exec()
 	return err
 }
 
+// DeleteGitOpenEuler delete data of ng
 func DeleteGitOpenEuler() error {
 	o := orm.NewOrm()
 	_, err := o.Raw("delete from cve_git_open_euler where source != ? ", "ng").Exec()
 	return err
 }
 
+// DeleteGitPackageInfo delete all data
 func DeleteGitPackageInfo() error {
 	o := orm.NewOrm()
 	_, err := o.Raw("delete from cve_git_package_info").Exec()
diff --git a/cve-vulner-manager/taskhandler/issuestatistics.go b/cve-vulner-manager/taskhandler/issuestatistics.go
index dd5d38e..48ef107 100644
--- a/cve-vulner-manager/taskhandler/issuestatistics.go
+++ b/cve-vulner-manager/taskhandler/issuestatistics.go
@@ -243,9 +243,11 @@ func IssueStatistics(beforeMonth, prcnum int, owner string) error {
 		}
 	}
 	zipFileName := excelPath
+
+	const deci = 10
 	if is.cveCount > 0 {
 		if templateId > 0 {
-			cBody := fmt.Sprint("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, 10) + "个, 其中" +
+			cBody := fmt.Sprint("hi all: \r\n 当前未解决漏洞有" + strconv.FormatInt(is.cveCount, deci) + "个, 其中" +
 				strconv.FormatInt(is.HighCveCount, 10) + "个7分以上漏洞, 详情见附件, 7分以上CVE请在周三之前解决. \r\n" +
 				"已经分析完毕的issue请maintainer尽快关掉, 否则影响数据统计; 提交PR时要关联issue, 若CVE在之前PR解决, 请PR提交人编辑PR信息将issue关联上. \r\n")
 			sendError := SendEmail(zipFileName, 2, cBody, "")
diff --git a/cve-vulner-manager/taskhandler/sendemail.go b/cve-vulner-manager/taskhandler/sendemail.go
index 9981807..218b822 100644
--- a/cve-vulner-manager/taskhandler/sendemail.go
+++ b/cve-vulner-manager/taskhandler/sendemail.go
@@ -370,7 +370,8 @@ func SendCvrfEmail(subFileSlice []string) {
 	}
 	subject := "The file name of the cvrf format currently to be manually reviewed is as follows.date: " + common.GetCurTime()
 	fileStr := strings.Join(subFileSlice, "\r\n")
-	body := fmt.Sprint("hi all: \r\n The list of cvrf format files that have been uploaded to the file server is as follows: \r\n" + fileStr)
+	body := fmt.Sprint("hi all: \r\n The list of cvrf format files that have been" +
+		" uploaded to the file server is as follows: \r\n" + fileStr)
 	sendErr := SendCommonMail(toEmailName, []string{}, subject, body, "")
 	if sendErr == nil {
 		logs.Info("Notify cvrf that the email was sent successfully!")
diff --git a/cve-vulner-manager/taskhandler/yaml.go b/cve-vulner-manager/taskhandler/yaml.go
index 39d8aa4..2cd8823 100644
--- a/cve-vulner-manager/taskhandler/yaml.go
+++ b/cve-vulner-manager/taskhandler/yaml.go
@@ -18,6 +18,7 @@ const (
 	pageSize   = 1000
 )
 
+// SyncEulerPackages sync packages of openeuler
 func SyncEulerPackages() error {
 	data, err := getEulerPackages()
 	if err != nil {
@@ -28,47 +29,54 @@ func SyncEulerPackages() error {
 		return err
 	}
 
+	handle := func(path string, detail []BranchDetail) {
+		originByVersion := groupByVersion(path, detail)
+		for _, origin := range originByVersion {
+			if err = saveData(origin); err != nil {
+				logs.Error("save data failed: ", err.Error())
+			}
+		}
+	}
+
 	for _, v := range data {
 		for k, detail := range v {
-			originByVersion := groupByVersion(k, detail)
-			for _, origin := range originByVersion {
-				if _, err = models.InsertEulerRepoOrigin(&origin); err != nil {
-					logs.Error("insert euler repo origin error:", err.Error())
-					continue
-				}
-
-				goeId, err := models.InsertYamlData(&models.GitOpenEuler{
-					PackageName: origin.PackageName,
-					Version:     origin.Version,
-					PackageId:   0,
-					TableId:     0,
-					Status:      1,
-					Source:      "openeuler",
-				})
-				if err != nil {
-					logs.Error("insert git_open_euler error:", err.Error())
-					continue
-				}
-
-				gpi := models.GitPackageInfo{
-					GitId:       goeId,
-					PackageName: origin.PackageName,
-					Version:     origin.Version,
-					Summary:     origin.Summary,
-					Decription:  origin.Decription,
-					Status:      0,
-				}
-
-				if _, err = models.InsertGitPackageInfo(&gpi); err != nil {
-					logs.Error("insert git_package_info error:", err.Error())
-				}
-			}
+			handle(k, detail)
 		}
 	}
 
 	return nil
 }
 
+func saveData(origin models.OpenEulerRepoOrigin) error {
+	if _, err := models.InsertEulerRepoOrigin(&origin); err != nil {
+		return err
+	}
+
+	goeId, err := models.InsertYamlData(&models.GitOpenEuler{
+		PackageName: origin.PackageName,
+		Version:     origin.Version,
+		PackageId:   0,
+		TableId:     0,
+		Status:      1,
+		Source:      "openeuler",
+	})
+	if err != nil {
+		return err
+	}
+
+	_, err = models.InsertGitPackageInfo(&models.GitPackageInfo{
+		GitId:       goeId,
+		PackageName: origin.PackageName,
+		Version:     origin.Version,
+		Summary:     origin.Summary,
+		Decription:  origin.Decription,
+		Status:      0,
+	})
+
+	return err
+}
+
+// BranchDetail struct of response data
 type BranchDetail struct {
 	Summary     string      `json:"summary"`
 	PackageName string      `json:"package_name"`
@@ -77,7 +85,7 @@ type BranchDetail struct {
 	BRName      string      `json:"brname"`
 }
 
-func (d BranchDetail) IsValid() bool {
+func (d BranchDetail) isValid() bool {
 	outSyncBranch := beego.AppConfig.String("cve::out_of_sync_branch")
 	outSyncBranchList := strings.Split(outSyncBranch, ",")
 	for _, v := range outSyncBranchList {
@@ -93,7 +101,7 @@ func (d BranchDetail) IsValid() bool {
 	return false
 }
 
-func (d BranchDetail) GenerateDescription() string {
+func (d BranchDetail) generateDescription() string {
 	switch d.Description.(type) {
 	case []interface{}:
 		var t []string
@@ -135,7 +143,7 @@ func groupByVersion(path string, detail []BranchDetail) map[string]models.OpenEu
 
 	originByVersion := make(map[string]models.OpenEulerRepoOrigin)
 	for _, d := range detail {
-		if !d.IsValid() {
+		if !d.isValid() {
 			continue
 		}
 
@@ -147,7 +155,7 @@ func groupByVersion(path string, detail []BranchDetail) map[string]models.OpenEu
 			}
 
 			if ov.Decription == "" {
-				ov.Decription = d.GenerateDescription()
+				ov.Decription = d.generateDescription()
 			}
 
 			originByVersion[d.Version] = ov
@@ -156,7 +164,7 @@ func groupByVersion(path string, detail []BranchDetail) map[string]models.OpenEu
 				PackageName: d.PackageName,
 				Version:     d.Version,
 				Summary:     d.Summary,
-				Decription:  d.GenerateDescription(),
+				Decription:  d.generateDescription(),
 				Branchs:     d.BRName,
 				Status:      1,
 				CreateTime:  common.GetCurTime(),
@@ -169,6 +177,7 @@ func groupByVersion(path string, detail []BranchDetail) map[string]models.OpenEu
 	return originByVersion
 }
 
+// Response struct of response
 type Response struct {
 	Code    int                         `json:"code"`
 	Message string                      `json:"message"`
@@ -176,7 +185,9 @@ type Response struct {
 }
 
 func getEulerPackages() ([]map[string][]BranchDetail, error) {
-	cli := utils.NewHttpClient(3)
+	const retryTimes = 3
+
+	cli := utils.NewHttpClient(retryTimes)
 	var result []map[string][]BranchDetail
 	page := 1
 	for {
-- 
Gitee


From 2edc1b7f83e098cb886f249cc8359c812787fa7c Mon Sep 17 00:00:00 2001
From: yangwei999 <348134071@qq.com>
Date: Mon, 12 May 2025 15:38:09 +0800
Subject: [PATCH 3/3] ignore webkit2gtk3

---
 cve-vulner-manager/taskhandler/yaml.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/cve-vulner-manager/taskhandler/yaml.go b/cve-vulner-manager/taskhandler/yaml.go
index 2cd8823..1a1e64a 100644
--- a/cve-vulner-manager/taskhandler/yaml.go
+++ b/cve-vulner-manager/taskhandler/yaml.go
@@ -94,6 +94,10 @@ func (d BranchDetail) isValid() bool {
 		}
 	}
 
+	if d.PackageName == "webkit2gtk3" {
+		return false
+	}
+
 	if d.PackageName != "" && d.Version != "" && !strings.Contains(d.Version, "%") {
 		return true
 	}
-- 
Gitee