From c5d06e8ae34a1a368de82436ab1587232342927a Mon Sep 17 00:00:00 2001 From: yangwei999 <348134071@qq.com> Date: Wed, 16 Jul 2025 10:38:08 +0800 Subject: [PATCH] optimize sync openeuler packages --- cve-vulner-manager/models/modeldb.go | 10 +++++----- cve-vulner-manager/models/ymal.go | 12 ++++++------ cve-vulner-manager/taskhandler/yaml.go | 16 +++++++--------- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/cve-vulner-manager/models/modeldb.go b/cve-vulner-manager/models/modeldb.go index 675a077..f61f0b0 100644 --- a/cve-vulner-manager/models/modeldb.go +++ b/cve-vulner-manager/models/modeldb.go @@ -254,9 +254,9 @@ type GitPackageTable struct { type GitOpenEuler struct { GitId int64 `orm:"pk;auto;column(git_id)"` - PackageId int64 `orm:"index;column(package_id)" description:"获取到的包id"` + PackageId int64 `orm:"column(package_id)" description:"获取到的包id"` PackageName string `orm:"column(package_name);size(256)" description:"包名称"` - Version string `orm:"size(64);column(version);index" description:"版本号"` + Version string `orm:"size(64);column(version)" description:"版本号"` Release string `orm:"size(128);column(releases)" description:"releases 版本号"` OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` License string `orm:"type(text);column(license)" description:"license"` @@ -268,7 +268,7 @@ type GitOpenEuler struct { LatestVersion string `orm:"size(128);column(latest_version)" description:"最新版本号"` LatestVersionTime string `orm:"size(32);column(latest_version_time)" description:"最新版本发布时间"` IssueCount int64 `orm:"column(issue_count);default(0)" description:"该软件包仓库下的issue总数"` - TableId int64 `orm:"index;column(table_id)" description:"表id, 外键"` + TableId int64 `orm:"column(table_id)" description:"表id, 外键"` TableName string `orm:"size(128);column(table_name)" description:"数据库pkginfo下的表名，如：mainline, bringInRely"` CpePackName string `orm:"size(512);column(cpe_packname);null" description:"openEuler对应的cpe包名称(或者影响的包名)"` Status int8 `orm:"default(0);column(status)" description:"0: 代表新数据; 1:已推送; 2:代表已废弃"` @@ -288,9 +288,9 @@ type GitOpenEulerTableRelate struct { type GitPackageInfo struct { DetailId int64 `orm:"pk;auto;column(detail_id)"` GitId int64 `orm:"index;column(git_id)" description:"包id，外键"` - Ids int64 `orm:"index;column(git_ids)" description:"原始数据id来源"` + Ids int64 `orm:"column(git_ids)" description:"原始数据id来源"` PackageName string `orm:"column(package_name);size(256)" description:"包名称"` - Version string `orm:"size(64);column(version);index" description:"版本号"` + Version string `orm:"size(64);column(version)" description:"版本号"` Release string `orm:"size(128);column(release)" description:"release 版本号"` OriginUrl string `orm:"size(512);column(origin_url)" description:"上游社区链接"` License string `orm:"type(text);column(license)" description:"license"` diff --git a/cve-vulner-manager/models/ymal.go b/cve-vulner-manager/models/ymal.go index 5bb3e3c..61e015e 100644 --- a/cve-vulner-manager/models/ymal.go +++ b/cve-vulner-manager/models/ymal.go @@ -35,23 +35,23 @@ func UpdateEulerRepoOrigin(ogk *OpenEulerRepoOrigin, fields ...string) error { } // DeleteEulerRepoOrigin delete all data -func DeleteEulerRepoOrigin() error { +func DeleteEulerRepoOrigin(now string) error { o := orm.NewOrm() - _, err := o.Raw("delete from cve_open_euler_repo_origin").Exec() + _, err := o.Raw("delete from cve_open_euler_repo_origin where create_time < ?", now).Exec() return err } // DeleteGitOpenEuler delete data of ng -func DeleteGitOpenEuler() error { +func DeleteGitOpenEuler(now string) error { o := orm.NewOrm() - _, err := o.Raw("delete from cve_git_open_euler where source != ? ", "ng").Exec() + _, err := o.Raw("delete from cve_git_open_euler where source != ? and create_time < ?", "ng", now).Exec() return err } // DeleteGitPackageInfo delete all data -func DeleteGitPackageInfo() error { +func DeleteGitPackageInfo(now string) error { o := orm.NewOrm() - _, err := o.Raw("delete from cve_git_package_info").Exec() + _, err := o.Raw("delete from cve_git_package_info where create_time < ?", now).Exec() return err } diff --git a/cve-vulner-manager/taskhandler/yaml.go b/cve-vulner-manager/taskhandler/yaml.go index 1a1e64a..d93961f 100644 --- a/cve-vulner-manager/taskhandler/yaml.go +++ b/cve-vulner-manager/taskhandler/yaml.go @@ -4,6 +4,7 @@ import ( "fmt" "net/http" "strings" + "time" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" @@ -25,10 +26,6 @@ func SyncEulerPackages() error { return err } - if err = cleanOldData(); err != nil { - return err - } - handle := func(path string, detail []BranchDetail) { originByVersion := groupByVersion(path, detail) for _, origin := range originByVersion { @@ -38,13 +35,14 @@ func SyncEulerPackages() error { } } + now := time.Now().Format(time.DateTime) for _, v := range data { for k, detail := range v { handle(k, detail) } } - return nil + return cleanOldData(now) } func saveData(origin models.OpenEulerRepoOrigin) error { @@ -121,16 +119,16 @@ func (d BranchDetail) generateDescription() string { } } -func cleanOldData() error { - if err := models.DeleteEulerRepoOrigin(); err != nil { +func cleanOldData(now string) error { + if err := models.DeleteEulerRepoOrigin(now); err != nil { return err } - if err := models.DeleteGitOpenEuler(); err != nil { + if err := models.DeleteGitOpenEuler(now); err != nil { return err } - if err := models.DeleteGitPackageInfo(); err != nil { + if err := models.DeleteGitPackageInfo(now); err != nil { return err } -- Gitee