From ae716bbd9fb5a150ed5b3a6c6012ecc38504cad0 Mon Sep 17 00:00:00 2001 From: sb Date: Thu, 8 Apr 2021 20:43:36 +0800 Subject: [PATCH] Add version processing logic in spec, delete historical yaml source version data --- conf/app.conf | 4 +- controllers/packages.go | 9 ++- controllers/upload.go | 1 + cve-py/controller/timertaskcontroller.py | 2 +- cve-py/tabletask/gauss_yaml.py | 95 +++++++++++------------ models/modeldb.go | 98 ++++++++++++------------ models/packages.go | 14 ++-- models/uploadcve.go | 6 +- models/ymal.go | 25 ++++-- task/yaml.go | 2 + taskhandler/assist.go | 1 - taskhandler/common.go | 2 +- taskhandler/cve.go | 7 +- taskhandler/grabissue.go | 2 +- taskhandler/oricvecheck.go | 1 + taskhandler/yaml.go | 79 +++++++++++++++---- util/time.go | 9 ++- 17 files changed, 210 insertions(+), 147 deletions(-) diff --git a/conf/app.conf b/conf/app.conf index e07ce76..197d709 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -49,9 +49,9 @@ maxsize=204800 [crontab] ymalflag = 2 -getymal = 0 35 11 * * * +getymal = 0 53 17 * * * eulerymalflag = 1 -eulergetymal = 0 42 16 * * * +eulergetymal = 0 50 11 * * * cveflag = 2 getcve = 0 14 11 * * * oricveflag = 2 diff --git a/controllers/packages.go b/controllers/packages.go index 87e9c4a..a8d2ceb 100644 --- a/controllers/packages.go +++ b/controllers/packages.go @@ -223,8 +223,15 @@ func (u *PackagesInfoController) Get() { resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) return } + version := u.GetString("version") + if version == "" { + logs.Error("version, Parameter error") + resp["errno"] = errcode.RecodeParamErr + resp["errmsg"] = errcode.RecodeText(errcode.RecodeParamErr) + return + } var gi models.GitPackageInfo - err := models.GetPackageInfo(pkgName, &gi) + err := models.GetPackageInfo(pkgName, version, &gi) if err != nil { resp["errno"] = errcode.RecodeNodata resp["errmsg"] = errcode.RecodeText(errcode.RecodeNodata) diff --git a/controllers/upload.go b/controllers/upload.go index f392cdb..f7f3282 100644 --- a/controllers/upload.go +++ b/controllers/upload.go @@ -257,6 +257,7 @@ func (u *UserUploadController) Post() { var gits models.GitOpenEuler gits.PackageName = pkList[0] gits.Version = pkList[1] + gits.Status = 1 ok := models.QueryCveOpeneulerdata(&gits) if ok { orCve.IsExit = 1 diff --git a/cve-py/controller/timertaskcontroller.py b/cve-py/controller/timertaskcontroller.py index 55986f6..06ed9b5 100644 --- a/cve-py/controller/timertaskcontroller.py +++ b/cve-py/controller/timertaskcontroller.py @@ -42,7 +42,7 @@ def timertask(): # parsing the table package_committer_excels scheduler.add_job(taskcontroller.run_package_committer_task, 'interval', hours=1) # Parse opengauss yaml file - scheduler.add_job(taskcontroller.parse_opengauss_yaml_task, 'cron', day_of_week='0-6', hour=8, minute=30) + scheduler.add_job(taskcontroller.parse_opengauss_yaml_task, 'cron', day_of_week='0-6', hour=2, minute=30) # Parse the issue statistics recipient list # scheduler.add_job(taskcontroller.issue_statistics_email_task, 'cron', day_of_week='0-6', hour=5, minute=30) # Complete the template information of the issue with the data on the CVE official website diff --git a/cve-py/tabletask/gauss_yaml.py b/cve-py/tabletask/gauss_yaml.py index b349e12..50cacce 100644 --- a/cve-py/tabletask/gauss_yaml.py +++ b/cve-py/tabletask/gauss_yaml.py @@ -68,37 +68,25 @@ def store_yaml_data(yaml_data): mysql = Mysql() if yaml_data is not None and len(yaml_data) > 0: # Delete all data before updating - # if True: - # delete_yaml_data(mysql) + delete_yaml_origin_data(mysql) + origin_id = 1 for yaml_key, yaml_value in yaml_data.items(): try: - origin_data = select_yaml_origin_data(yaml_key, yaml_value, mysql) + origin_data = select_yaml_origin_data(yaml_key, yaml_value["version"], mysql) if origin_data: update_yaml_origin_data(origin_data["id"], yaml_value, mysql) else: - insert_yaml_origin_data(yaml_key, yaml_value, mysql) + insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id) + origin_id += 1 packages_data = select_yaml_data(yaml_key, yaml_value, mysql) if packages_data is not None: - update_flag = False - for pk in packages_data: - if pk["package_id"] > 0: - update_flag = True - break - # if pk["package_id"] >= 10000000: - # update_flag = True - # update_yaml_data(pk["package_id"], yaml_key, yaml_value, mysql) - # update_yaml_detail_data(pk["git_id"], yaml_key, yaml_value, mysql) - # mysql.end() - # break - if not update_flag: - package_ids = select_yaml_lastdata() - if package_ids is not None and package_ids["package_id"] >= 10000000: - packg_id = add_package_id(package_ids["package_id"]) - else: - packg_id = add_package_id(0) - last_id = insert_yaml_data(packg_id, yaml_key, yaml_value, mysql) - insert_yaml_detail_data(last_id, yaml_key, yaml_value, mysql) - mysql.end() + if len(packages_data) > 1: + for pd in packages_data[1:]: + delete_yaml_openeuler_detail_data(mysql, pd["git_id"]) + delete_yaml_openeuler_data(mysql, pd["git_id"]) + pk = packages_data[0] + update_yaml_data(pk["package_id"], yaml_key, yaml_value, mysql) + update_yaml_detail_data(pk["git_id"], yaml_key, yaml_value, mysql) else: package_ids = select_yaml_lastdata() if package_ids is not None and package_ids["package_id"] >= 10000000: @@ -107,36 +95,39 @@ def store_yaml_data(yaml_data): packg_id = add_package_id(0) last_id = insert_yaml_data(packg_id, yaml_key, yaml_value, mysql) insert_yaml_detail_data(last_id, yaml_key, yaml_value, mysql) - mysql.end() - except pymysql.err.IntegrityError: - print(pymysql.err.IntegrityError) - # except Exception as e: - # print(e) + mysql.dispose() + # except pymysql.err.IntegrityError: + # print(pymysql.err.IntegrityError) + except Exception as e: + print(e) mysql.dispose(2) mysql.close() -def delete_yaml_data(mysql): +def delete_yaml_data(): """ delete yaml data """ - delete_yaml_origin_data(mysql) + mysql = Mysql() # Delete historical data packages_data = select_openeuler_yaml_data(mysql) if packages_data is not None: for pk in packages_data: print("pk: ", pk) if pk["package_id"] >= 10000000: - print("ID of the currently deleted data: ", pk["git_id"]) - delete_yaml_openeuler_detail_data(mysql, pk["git_id"]) - delete_yaml_openeuler_data(mysql, pk["git_id"]) + origin_data = select_yaml_origin_data(pk["package_name"], pk["version"], mysql) + if not origin_data: + print("ID of the currently deleted data: ", pk["git_id"]) + delete_yaml_openeuler_detail_data(mysql, pk["git_id"]) + delete_yaml_openeuler_data(mysql, pk["git_id"]) + mysql.close() def select_openeuler_yaml_data(mysql): """ Query data """ - sql = "SELECT package_id, git_id FROM cve_git_open_euler " \ + sql = "SELECT package_id, git_id,package_name,version FROM cve_git_open_euler " \ "WHERE package_id >= %s" val = (10000000,) packages_data = mysql.getMany(sql, val) @@ -171,11 +162,11 @@ def insert_yaml_data(package_id, yaml_key, yaml_value, mysql): insert data """ insert_sql = "insert into cve_git_open_euler(package_id,package_name," \ - "version,origin_url,create_time,update_time,cpe_packname) " \ - "values(%s,%s,%s,%s,%s,%s,%s)" + "version,origin_url,create_time,update_time,cpe_packname,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" val = (package_id, yaml_key, yaml_value["version"], yaml_value["url"], times.get_current_time(), - times.get_current_time(), yaml_value["cpeName"]) + times.get_current_time(), yaml_value["cpeName"], 1) last_id = mysql.insertOne(insert_sql, val) return last_id @@ -185,11 +176,11 @@ def insert_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): insert data details """ insert_sql = "insert into cve_git_package_info(git_id,package_name," \ - "version,origin_url,create_time,update_time,decription) " \ - "values(%s,%s,%s,%s,%s,%s,%s)" + "version,origin_url,create_time,update_time,decription,status) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" val = (pg_detail_id, yaml_key, yaml_value["version"], yaml_value["url"], times.get_current_time(), - times.get_current_time(), "") + times.get_current_time(), "", 0) last_id = mysql.insertOne(insert_sql, val) return last_id @@ -199,10 +190,10 @@ def update_yaml_data(package_id, yaml_key, yaml_value, mysql): update data """ update_sql = "update cve_git_open_euler set origin_url = %s,update_time = %s, " \ - "cpe_packname = %s where package_id = %s " \ + "cpe_packname = %s,status=%s where package_id = %s " \ "and package_name =%s and version = %s" val = (yaml_value["url"], times.get_current_time(), yaml_value["cpeName"], - package_id, yaml_key, yaml_value["version"]) + 1, package_id, yaml_key, yaml_value["version"]) mysql.update(update_sql, val) @@ -211,9 +202,9 @@ def update_yaml_detail_data(pg_detail_id, yaml_key, yaml_value, mysql): Update data details """ update_sql = "update cve_git_package_info set package_name = %s,version = %s, " \ - "origin_url = %s, update_time = %s where git_id = %s" + "origin_url = %s, update_time = %s,status=%s where git_id = %s" val = (yaml_key, yaml_value["version"], yaml_value["url"], - times.get_current_time(), pg_detail_id) + times.get_current_time(), 0, pg_detail_id) mysql.update(update_sql, val) @@ -229,16 +220,16 @@ def update_yaml_origin_data(pg_origin_id, yaml_value, mysql): mysql.dispose() -def insert_yaml_origin_data(yaml_key, yaml_value, mysql): +def insert_yaml_origin_data(yaml_key, yaml_value, mysql, origin_id): """ insert origin data """ insert_sql = "insert into cve_open_guss_yaml(package_name," \ - "version,origin_url, status, cpe_name,create_time,update_time) " \ - "values(%s,%s,%s,%s,%s,%s, %s)" + "version,origin_url, status, cpe_name,create_time,update_time, id) " \ + "values(%s,%s,%s,%s,%s,%s,%s,%s)" val = (yaml_key, yaml_value["version"], yaml_value["url"], 1, yaml_value["cpeName"], times.get_current_time(), - times.get_current_time()) + times.get_current_time(), origin_id) last_id = mysql.insertOne(insert_sql, val) mysql.dispose() return last_id @@ -268,13 +259,13 @@ def delete_yaml_origin_data(mysql): mysql.dispose() -def select_yaml_origin_data(yaml_key, yaml_value, mysql): +def select_yaml_origin_data(package_name, version, mysql): """ Query origin data """ sql = "SELECT id FROM cve_open_guss_yaml WHERE package_name = %s " \ "and version = %s" - val = (yaml_key, yaml_value["version"]) + val = (package_name, version) packages_data = mysql.getOne(sql, val) if packages_data: return packages_data @@ -310,3 +301,5 @@ def proc_gauss_yaml(): store_yaml_data(yaml_data) else: print("File download failed, file: ", file_name) + # delete history yaml_data + delete_yaml_data() diff --git a/models/modeldb.go b/models/modeldb.go index 259726c..ffc4b7f 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -36,17 +36,17 @@ type AdminUser struct { type VulnCenter struct { CveId int64 `orm:"pk;auto;column(cve_id)"` CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` - Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` + Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求); 5:issue已经创建过，不符合要求，不处理;6:字段为空(评分,描述等);7:issue已被删除不再创建"` CveVersion string `orm:"size(512);column(cve_version)" description:"cve归属版本,版本合并"` RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` - CveUrl string `orm:"size(2048);column(cve_url)" description:"cve下载链接"` + CveUrl string `orm:"type(text);column(cve_url)" description:"cve下载链接"` IsExport int8 `orm:"default(0);column(is_export)" description:"0:数据初始化;1:数据已同步到官网；2：数据不需要同步到官网；3：issue已关闭需要同步到官网"` DataSource int8 `orm:"default(1);column(data_source)" description:"1:软件所提供;2:网络爬取；3:人工处理; 4:兼容之前的数据"` - CveDetailUrl string `orm:"size(1024);column(cve_detail_url)" description:"cve详情链接"` + CveDetailUrl string `orm:"type(text);column(cve_detail_url)" description:"cve详情链接"` OrganizationID int8 `orm:"default(1);column(organizate_id)" description:"默认值为:1,1:来源openEuler;2:来源opengauss;3:两者都有"` CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` @@ -115,11 +115,11 @@ type SecurityNotice struct { OpenId int64 `orm:"unique;column(openeuler_id);null" description:"OpenEulerSA"` Introduction string `orm:"size(256);null;column(introduction)" description:"简介"` Summary string `orm:"size(256);null;column(summary)" description:"概要"` - Theme string `orm:"size(2048);null;column(theme)" description:"主题"` - Description string `orm:"size(8192);column(description)" description:"安全公告描述"` + Theme string `orm:"type(text);null;column(theme)" description:"主题"` + Description string `orm:"type(text);column(description)" description:"安全公告描述"` InfluenceComponent string `orm:"size(256);null;column(influence_component)" description:"影响组件"` AffectProduct string `orm:"size(256);null;column(affect_product)" description:"影响产品"` - ReferenceLink string `orm:"size(1024);null;column(reference_link)" description:"参考链接"` + ReferenceLink string `orm:"type(text);null;column(reference_link)" description:"参考链接"` Status int8 `orm:"default(0);column(sec_status)" description:"0:未发布；1:已发布"` AffectStatus string `orm:"size(16);null;column(affect_status)" description:"Fixed:已解决；UnFixed:未解决；UnAffected:不影响"` CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` @@ -130,8 +130,8 @@ type SecurityNotice struct { type Package struct { Id int64 `orm:"pk;auto"` SecId int64 `orm:"index;column(sec_id)"` - PackName string `orm:"size(1024);null;column(pack_name)" description:"包名"` - PackUrl string `orm:"size(2048);null;column(pack_url)" description:"包下载链接"` + PackName string `orm:"type(text);null;column(pack_name)" description:"包名"` + PackUrl string `orm:"type(text);null;column(pack_url)" description:"包下载链接"` AffectProduct string `orm:"size(256);null;column(affect_product)" description:"影响产品"` Source string `orm:"size(128);null;column(source)" description:"数据源"` } @@ -146,11 +146,11 @@ type IssueTemplate struct { OpenEulerScore float64 `orm:"digits(10);decimals(1);column(openeuler_score)" description:"openeuler评分"` NVDVector string `orm:"size(256);column(nvd_vector)" description:"nvd评分向量"` OpenEulerVector string `orm:"size(256);column(openeuler_vector)" description:"openeuler评分向量"` - CveBrief string `orm:"size(4096);column(cve_brief)" description:"漏洞简述"` - CveAnalysis string `orm:"size(4096);column(cve_analysis)" description:"影响性分析说明"` - PrincipleAnalysis string `orm:"size(4096);column(principle_analysis)" description:"原理分析"` + CveBrief string `orm:"type(text);column(cve_brief)" description:"漏洞简述"` + CveAnalysis string `orm:"type(text);column(cve_analysis)" description:"影响性分析说明"` + PrincipleAnalysis string `orm:"type(text);column(principle_analysis)" description:"原理分析"` AffectedVersion string `orm:"size(256);column(affected_version)" description:"受影响的版本"` - Solution string `orm:"size(1024);column(solution)" description:"规避方案或消减措施"` + Solution string `orm:"type(text);column(solution)" description:"规避方案或消减措施"` IssueId int64 `orm:"column(issue_id)" description:"issue的id"` IssueNum string `orm:"size(64);column(issue_num)" description:"issue编号"` Assignee string `orm:"size(128);column(issue_assignee)" description:"issue所属责任人"` @@ -218,7 +218,7 @@ type GitOpenEuler struct { Version string `orm:"size(64);column(version);index" description:"版本号"` Release string `orm:"size(128);column(releases)" description:"releases 版本号"` OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` - License string `orm:"size(4096);column(license)" description:"license"` + License string `orm:"type(text);column(license)" description:"license"` Feature string `orm:"size(128);column(feature)" description:"特性"` MainTainer string `orm:"size(128);column(main_tainer)" description:"维护人"` MainTainLevel int8 `orm:"column(main_tain_level);default(1)" description:"软件包维护级别"` @@ -251,14 +251,14 @@ type GitPackageInfo struct { Version string `orm:"size(64);column(version);index" description:"版本号"` Release string `orm:"size(128);column(release)" description:"release 版本号"` OriginUrl string `orm:"size(512);column(origin_url)" description:"上游社区链接"` - License string `orm:"size(4096);column(license)" description:"license"` + License string `orm:"type(text);column(license)" description:"license"` Feature string `orm:"size(128);column(feature)" description:"特性"` MainTainer string `orm:"size(128);column(main_tainer)" description:"维护人"` MainTainLevel int8 `orm:"column(main_tain_level);default(1)" description:"软件包维护级别"` GitUrl string `orm:"size(512);column(git_url)" description:"gitee上的地址"` - Summary string `orm:"size(1024);column(summary)" description:"summary"` - Decription string `orm:"size(8092);column(decription)" description:"decription"` - BuildRequired string `orm:"size(4096);column(build_require)" description:"源码包的编译依赖"` + Summary string `orm:"type(text);column(summary)" description:"summary"` + Decription string `orm:"type(text);column(decription)" description:"decription"` + BuildRequired string `orm:"type(text);column(build_require)" description:"源码包的编译依赖"` Status int8 `orm:"default(0);column(status)" description:"0: 代表新数据; 1:已推送; 2:代表已废弃"` CreateTime time.Time `orm:"auto_now_add;type(datetime);column(create_time)"` UpdateTime time.Time `orm:"auto_now;type(datetime);column(update_time)"` @@ -277,7 +277,7 @@ type GitSubPackRequire struct { SubId int64 `orm:"index;column(sub_id)" description:"包id,外键"` Ids int64 `orm:"index;column(ids)" description:"原始数据id来源"` RequireName string `orm:"size(256);column(require_name)" description:"二进制包名"` - Providedby string `orm:"size(4096);column(providedb)" description:"提供该组件的二进制包"` + Providedby string `orm:"type(text);column(providedb)" description:"提供该组件的二进制包"` } type GitSubPackProvides struct { @@ -301,7 +301,7 @@ type OriginUpstream struct { UpdateType string `orm:"size(32);column(update_type);" description:"数据上传类型：insert, update, delete"` CvePackName string `orm:"size(512);column(cve_packname);null" description:"Cve在上游对应的包名"` PackName string `orm:"size(512);column(git_packname);index;null" description:"Cve对应的openEuler包名称(或者影响的包名)"` - Title string `orm:"size(1024);column(cve_title);null" description:"标题"` + Title string `orm:"type(text);column(cve_title);null" description:"标题"` AffectProduct string `orm:"size(512);column(affect_porduct);null" description:"Cve影响的组件, 对应"` CnnvdID string `orm:"size(256);column(cnnvd_id);null" description:"Cnnvd_id"` CnvdID string `orm:"size(256);column(cnvd_id);null" description:"Cnvd_id"` @@ -327,8 +327,8 @@ type OriginUpstream struct { type OriginUpstreamDesc struct { DescId int64 `orm:"pk;auto;column(desc_id)"` CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` - EnDescription string `orm:"size(8192);column(en_desc);null" description:"cve英文描述"` - ZhDescription string `orm:"size(8192);column(zh_desc);null" description:"cve中文描述"` + EnDescription string `orm:"type(text);column(en_desc);null" description:"cve英文描述"` + ZhDescription string `orm:"type(text);column(zh_desc);null" description:"cve中文描述"` } type OriginUpstreamConfig struct { @@ -357,8 +357,8 @@ type IssueAssignee struct { type OriginUpstreamConfigNodeCpe struct { CpeId int64 `orm:"pk;auto;column(cpe_id)"` NodeId int64 `orm:"index;column(node_id)" description:"OriginUpstreamConfigNode 外键"` - Cpe23Uri string `orm:"size(1024);column(cpe_uri);null" description:"cpe"` - CpeMatchString string `orm:"size(1024);column(cpe_match);null" description:"过滤字符串"` + Cpe23Uri string `orm:"type(text);column(cpe_uri);null" description:"cpe"` + CpeMatchString string `orm:"type(text);column(cpe_match);null" description:"过滤字符串"` Vulnerable string `orm:"size(64);column(vulner_able);null" description:"易受攻击"` } @@ -429,7 +429,7 @@ type OriginUpstreamPoc struct { Path string `orm:"size(512);column(path);null" description:"文件路径"` Dbindex string `orm:"size(512);column(dbindex);null" description:"数据库索引"` Url string `orm:"size(512);column(url);null" description:"下载链接"` - Desc string `orm:"size(2048);column(desc);null" description:"说明"` + Desc string `orm:"type(text);column(desc);null" description:"说明"` } type OriginUpstreamEvent struct { @@ -438,7 +438,7 @@ type OriginUpstreamEvent struct { Title string `orm:"size(512);column(title);null" description:"标题"` Date string `orm:"size(32);column(date);null" description:"日期"` Url string `orm:"size(512);column(url);null" description:"连接"` - Description string `orm:"size(2048);column(description);null" description:"说明"` + Description string `orm:"type(text);column(description);null" description:"说明"` } type OriginUpstreamReference struct { @@ -447,29 +447,29 @@ type OriginUpstreamReference struct { Name string `orm:"size(512);column(name);null" description:"名称"` Refsource string `orm:"size(512);column(refsource);null" description:"参考来源"` Url string `orm:"size(512);column(url);null" description:"链接"` - Tags string `orm:"size(2048);column(tags);null" description:"tags"` + Tags string `orm:"type(text);column(tags);null" description:"tags"` } type OriginUpstreamVulType struct { VulId int64 `orm:"pk;auto;column(vul_id)"` CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` Cwe string `orm:"size(256);column(cwe);null" description:"Cwe编号"` - EnDesc string `orm:"size(2048);column(en_desc);null" description:"英文描述"` - ZhDesc string `orm:"size(2048);column(zh_desc);null" description:"中文描述"` + EnDesc string `orm:"type(text);column(en_desc);null" description:"英文描述"` + ZhDesc string `orm:"type(text);column(zh_desc);null" description:"中文描述"` } type OriginUpstreamFixSuggest struct { FixId int64 `orm:"pk;auto;column(fix_id)"` CveId int64 `orm:"index;column(cve_id)" description:"OriginUpstream 外键"` - Detail string `orm:"size(1024);column(detail);null" description:"细节"` + Detail string `orm:"type(text);column(detail);null" description:"细节"` } type OriginUpstreamFixSuggestRef struct { FixRefId int64 `orm:"pk;auto;column(fix_ref_id)"` FixId int64 `orm:"index;column(fix_id)" description:"OriginUpstreamFixSuggest 外键"` Refsource string `orm:"size(512);column(refsource);null" description:"参考文献，包含以下内容"` - Url string `orm:"size(1024);column(url);null" description:"链接"` - Name string `orm:"size(1024);column(name);null" description:"名称"` + Url string `orm:"type(text);column(url);null" description:"链接"` + Name string `orm:"type(text);column(name);null" description:"名称"` } type OriginUpstreamFixSuggestRefTag struct { @@ -508,13 +508,13 @@ type GiteOriginIssue struct { type OriginExcel struct { CveId int64 `orm:"pk;auto;column(cve_id)"` CveNum string `orm:"size(256);column(cve_num)" description:"cve编号"` - CveUrl string `orm:"size(2048);column(cve_url);null" description:"cve下载链接"` + CveUrl string `orm:"type(text);column(cve_url);null" description:"cve下载链接"` CveVersion string `orm:"size(256);column(cve_version);null" description:"漏洞归属版本"` PackName string `orm:"size(256);column(pack_name);null" description:"包名"` ScoreType string `orm:"size(16);column(score_type);null" description:"评分类型：v3"` NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score);null" description:"nvd评分"` CveLevel string `orm:"size(32);column(cve_level);null" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` - CveDesc string `orm:"size(4096);column(cve_desc);null" description:"漏洞简述"` + CveDesc string `orm:"type(text);column(cve_desc);null" description:"漏洞简述"` RepairTime string `orm:"size(32);column(repair_time);null" description:"cve发布时间"` NVDVector string `orm:"size(256);column(vector_value);null" description:"nvd 评分向量"` AttackVector string `orm:"size(256);column(attack_vector);null" description:"nvd 3.0评分"` @@ -612,7 +612,7 @@ type IssueCreateRecord struct { IssueRecordId int64 `orm:"pk;auto;column(id)"` CveId int64 `orm:"index;column(cve_id)" description:"VulnCenter 外键"` CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` - Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` + Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` Status int8 `orm:"default(1);column(status)" description:"1:已创建issue；2：未创建issue；3：创建失败; 4: 已导出"` CveVersion string `orm:"size(128);column(cve_version)" description:"cve归属版本"` @@ -620,7 +620,7 @@ type IssueCreateRecord struct { PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd 评分"` NvectorVule string `orm:"size(256);column(n_vector_value)" description:"nvd vector 原始值"` - ErrorDescription string `orm:"size(1024);column(error_description);null" description:"创建cve错误信息反馈"` + ErrorDescription string `orm:"type(text);column(error_description);null" description:"创建cve错误信息反馈"` CreateTime string `orm:"size(32);column(create_time)"` UpdateTime string `orm:"size(32);column(update_time);null"` DeleteTime string `orm:"size(32);column(delete_time);null"` @@ -632,7 +632,7 @@ type OpenGussYaml struct { Version string `orm:"size(64);column(version);index" description:"版本号"` OriginUrl string `orm:"size(512);column(origin_url)" description:"gitee上的地址"` Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` - CpeName string `orm:"size(1024);column(cpe_name)" description:"cpe的名称"` + CpeName string `orm:"type(text);column(cpe_name)" description:"cpe的名称"` CreateTime string `orm:"size(32);column(create_time)"` UpdateTime string `orm:"size(32);column(update_time);null"` DeleteTime string `orm:"size(32);column(delete_time);null"` @@ -651,7 +651,7 @@ type SpecIssueAssigness struct { type SpecError struct { Id int64 `orm:"pk;auto;column(id)"` CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` - Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` + Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` Owner string `orm:"size(256);column(cve_owner);index" description:"仓库所在组织"` Status int8 `orm:"default(1);column(cve_status)" description:"1:正常数据;2:已删除"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` @@ -664,8 +664,8 @@ type CvrfSaRecord struct { Id int64 `orm:"pk;auto;column(id)"` OpenEulerSANum string `orm:"size(128);column(openeuler_sa_num);unique" description:"安全公告"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` - CveNum string `orm:"size(8192);column(cve_num);index" description:"cve编号"` - XmlContent string `orm:"text;column(xml_content)" description:"xml内容"` + CveNum string `orm:"type(text);column(cve_num);index" description:"cve编号"` + XmlContent string `orm:"type(text);column(xml_content)" description:"xml内容"` Md5 string `orm:"size(128);column(cur_md5);unique" description:"当前内容生成的MD5"` UpdateMd5 string `orm:"size(128);column(update_md5)" description:"内容更新后生成的MD5"` Status int8 `orm:"default(1);column(cve_status)" description:"1:初始化; 2: 已更新; 3: 已删除"` @@ -723,12 +723,12 @@ type OpenGaussSiteList struct { CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` CveLevelValue int `orm:"default(0);column(cve_level_value)" desription:"Critical: 1, High: 2, Medium: 3,low: 4"` Summary string `orm:"size(256);null;column(summary)" description:"概要"` - Theme string `orm:"size(2048);null;column(theme)" description:"主题"` + Theme string `orm:"type(text);null;column(theme)" description:"主题"` Description string `orm:"size(8192);column(description)" description:"安全公告描述"` InfluenceComponent string `orm:"size(256);null;column(influence_component)" description:"影响组件"` AffectProduct string `orm:"size(256);null;column(affect_product)" description:"影响产品"` - CveNums string `orm:"size(2048);column(cve_nums)" description:"cve编号汇总"` - ReferenceLink string `orm:"size(1024);null;column(reference_link)" description:"参考链接"` + CveNums string `orm:"type(text);column(cve_nums)" description:"cve编号汇总"` + ReferenceLink string `orm:"type(text);null;column(reference_link)" description:"参考链接"` PublicDate string `orm:"size(32);null;column(public_date)" description:"sa发布日期"` Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除;3:已发布"` Years int `orm:"null;column(sa_years)" description:"sa 发布年份"` @@ -741,7 +741,7 @@ type OpenGaussCveList struct { Id int64 `orm:"pk;auto"` GaussId int64 `orm:"index;column(gauss_id)"` CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` - Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` + Description string `orm:"type(text);column(cve_desc)" description:"cve描述"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd 评分"` OpenEulerScore float64 `orm:"digits(10);decimals(1);column(openeuler_score)" description:"openeuler评分"` @@ -791,7 +791,7 @@ type OpenGaussPackage struct { PlatId int64 `orm:"index;column(plat_id)"` GaussId int64 `orm:"index;column(gauss_id);default(0)" description:"对应sa的表id"` PackageName string `orm:"size(512);column(package_name)" description:"cve对应得包名称"` - RpmName string `orm:"size(1024);column(rpm_name)" description:"tag包名称"` + RpmName string `orm:"type(text);column(rpm_name)" description:"tag包名称"` ReleaseDate string `orm:"size(32);column(release_date)" description:"版本发布日期"` RepoVersion string `orm:"size(32);column(repo_version);" description:"分支版本信息"` Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` @@ -846,12 +846,12 @@ type OpenGaussListTemp struct { CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` CveLevelValue int `orm:"default(0);column(cve_level_value)" desription:"Critical: 1, High: 2, Medium: 3,low: 4"` Summary string `orm:"size(256);null;column(summary)" description:"概要"` - Theme string `orm:"size(2048);null;column(theme)" description:"主题"` - Description string `orm:"size(8192);column(description)" description:"安全公告描述"` + Theme string `orm:"type(text);null;column(theme)" description:"主题"` + Description string `orm:"type(text);column(description)" description:"安全公告描述"` InfluenceComponent string `orm:"size(256);null;column(influence_component)" description:"影响组件"` AffectProduct string `orm:"size(256);null;column(affect_product)" description:"影响产品"` PublicDate string `orm:"size(32);null;column(public_date)" description:"sa发布日期"` - CveNums string `orm:"size(2048);column(cve_nums)" description:"cve编号汇总"` + CveNums string `orm:"type(text);column(cve_nums)" description:"cve编号汇总"` Status int8 `orm:"default(1);column(status)" description:"1:正常可用;2:已删除"` CreateTime string `orm:"size(32);column(create_time)"` UpdateTime string `orm:"size(32);column(update_time);null"` @@ -862,8 +862,8 @@ type OpenEulerRepoOrigin struct { Id int64 `orm:"pk;auto;column(id)"` PackageName string `orm:"column(package_name);size(256)" description:"包名称"` Version string `orm:"size(64);column(version)" description:"版本号"` - Summary string `orm:"size(1024);column(summary)" description:"简介"` - Decription string `orm:"size(8092);column(decription)" description:"描述"` + Summary string `orm:"type(text);column(summary)" description:"简介"` + Decription string `orm:"type(text);column(decription)" description:"描述"` Branchs string `orm:"size(512);column(branch)" description:"分支信息"` Status int8 `orm:"default(1);column(status)" description:"1:正常;2:已删除"` CreateTime string `orm:"size(32);column(create_time)"` diff --git a/models/packages.go b/models/packages.go index 3251b05..78b556d 100644 --- a/models/packages.go +++ b/models/packages.go @@ -17,10 +17,10 @@ func GetIpWhite(ip string, iw *IpWhite) (err error) { return nil } -func GetPackageInfo(packageName string, gi *GitPackageInfo) (err error) { +func GetPackageInfo(packageName, version string, gi *GitPackageInfo) (err error) { o := orm.NewOrm() - err = o.Raw("select * FROM cve_git_package_info WHERE package_name=?", - packageName).QueryRow(gi) + err = o.Raw("select * FROM cve_git_package_info WHERE package_name=? and version=? and status=?", + packageName, version, 0).QueryRow(gi) if err == nil { logs.Info("GetPackageInfo, cve_git_package_info, "+ "packageName: ", packageName, ", search successful") @@ -104,7 +104,7 @@ func GetPackageList(pagesize int64, pagenum int64, QueryPkgName string) (ge []Gi qs := o.QueryTable("cve_git_open_euler") var us []GitOpenEuler if QueryPkgName != "" { - cnt, err := qs.Filter("package_name__icontains", QueryPkgName).Limit(pagesize, (pagenum-1)*pagesize).All(&us) + cnt, err := qs.Filter("package_name__icontains", QueryPkgName).Filter("status", 1).Limit(pagesize, (pagenum-1)*pagesize).All(&us) if err == nil { logs.Info("GetPackageList, cve_git_open_euler, count: ", cnt, ",pagesize ", pagesize, ",pagenum ", pagenum, ", QueryPkgName ", QueryPkgName) @@ -114,7 +114,7 @@ func GetPackageList(pagesize int64, pagenum int64, QueryPkgName string) (ge []Gi } return us, cnt, err } else { - cnt, err := qs.Limit(pagesize, (pagenum-1)*pagesize).All(&us) + cnt, err := qs.Filter("status", 1).Limit(pagesize, (pagenum-1)*pagesize).All(&us) if err == nil { logs.Info("GetPackageList, cve_git_open_euler, count: ", cnt, ",pagesize ", pagesize, ",pagenum ", pagenum, ", QueryPkgName ", QueryPkgName) @@ -131,14 +131,14 @@ func GetPackageNum(QueryPkgName string) int64 { cg := o.QueryTable("cve_git_open_euler") var ge []GitOpenEuler if QueryPkgName != "" { - num, err := cg.Filter("package_name__icontains", QueryPkgName).All(&ge) + num, err := cg.Filter("package_name__icontains", QueryPkgName).Filter("status", 1).All(&ge) if err == nil { return num } else { return 0 } } else { - num, err := cg.Filter("git_id__gt", 0).All(&ge) + num, err := cg.Filter("git_id__gt", 0).Filter("status", 1).All(&ge) if err == nil { return num } else { diff --git a/models/uploadcve.go b/models/uploadcve.go index 6711dd7..6d4b21e 100644 --- a/models/uploadcve.go +++ b/models/uploadcve.go @@ -166,7 +166,7 @@ func QueryRepoMaintainer(pkgName string) (string, bool) { } func QueryCveOpeneulerdata(ge *GitOpenEuler) bool { - geErr := QueryYamlData(ge, "PackageName", "Version") + geErr := QueryYamlData(ge, "PackageName", "Version", "Status") if ge.GitId == 0 { logs.Error("QueryCveOpeneulerdata, There is no corresponding repo or version, ", ge.PackageName, ge.Version, ",geErr: ", geErr) @@ -178,8 +178,8 @@ func QueryCveOpeneulerdata(ge *GitOpenEuler) bool { func QueryCveOpeneulerDetaildataByName(pkgName, versions string) (GitPackageInfo, bool) { o := orm.NewOrm() var goe GitPackageInfo - err := o.Raw("select * from cve_git_package_info where package_name = ? and version = ? "+ - "order by detail_id limit 1", pkgName, versions).QueryRow(&goe) + err := o.Raw("select * from cve_git_package_info where package_name = ? and version = ? and status = ?"+ + "order by detail_id limit 1", pkgName, versions, 0).QueryRow(&goe) if err != nil { logs.Info("QueryCveOpeneulerDetaildataByName, cve_git_package_info, pkgName: ", pkgName, ", versions: ", versions, ", No corresponding data") diff --git a/models/ymal.go b/models/ymal.go index 07080a5..3853a0e 100644 --- a/models/ymal.go +++ b/models/ymal.go @@ -61,7 +61,7 @@ func CreateYamlTable(gt *GitPackageTable) (TableId int64, err error) { func GetSingleYaml(ge *GitOpenEuler) (bool, error) { o := orm.NewOrm() if ge.Version != "" && ge.Release != "" { - localge := GitOpenEuler{PackageName: ge.PackageName, Version: ge.Version} + localge := GitOpenEuler{PackageName: ge.PackageName, Version: ge.Version, Status: 1} err := o.Read(&localge, "PackageName", "Version", "Release") logs.Info("GetSingleYaml1, Query yaml data results: ", ge, ", localge: ", localge, ", err: ", err) @@ -73,7 +73,7 @@ func GetSingleYaml(ge *GitOpenEuler) (bool, error) { return true, err } } else { - localge := GitOpenEuler{PackageName: ge.PackageName} + localge := GitOpenEuler{PackageName: ge.PackageName, Status: 1} err := o.Read(&localge, "PackageName") logs.Info("GetSingleYaml2, Query yaml data results: ", ge, ", localge: ", localge, ", err: ", err) @@ -129,7 +129,7 @@ func CreateYaml(ge *GitOpenEuler, tableValue GitPackageTable) (id int64, typex s ge.PackageName, ge.Version).QueryRow(&localge) if errx != nil { logs.Info("createYaml: ", localge, ",errx: ", errx) - ge.Status = 0 + ge.Status = 1 var GitId int64 if GitId, err = o.Insert(ge); err == nil { logs.Info("CreateYaml, insert cve_git_open_euler success, GitId: ", @@ -297,10 +297,21 @@ func UpdateEulerRepoOrigin(ogk *OpenEulerRepoOrigin, fields ...string) error { func GetEulerYamlList(yamlId int64, count int) (oer []OpenEulerRepoOrigin) { o := orm.NewOrm() - _, errx := o.Raw("select *" + - " from cve_open_euler_repo_origin where id > ? order by id desc limit ?", yamlId, count).QueryRows(&oer) - if errx != nil { - logs.Error("GetEulerYamlList, Get data does not exist, err: ", errx) + num, geErr := o.Raw("select *"+ + " from cve_open_euler_repo_origin where id > ? order by id asc limit ?", yamlId, count).QueryRows(&oer) + if num == 0 { + logs.Error("GetEulerYamlList, Get data does not exist, geErr: ", geErr) + } + return +} + +func GetGitOpenEulerList(gitId int64, count int) (goe []GitOpenEuler) { + o := orm.NewOrm() + num, geErr := o.Raw("select *"+ + " from cve_git_open_euler where git_id > ? and package_id < ? and status = 1 order by git_id asc limit ?", + gitId, 10000000, count).QueryRows(&goe) + if num == 0 { + logs.Error("GetGitOpenEulerList, Get data does not exist, geErr: ", geErr) } return } diff --git a/task/yaml.go b/task/yaml.go index 3b21df9..7835ba2 100644 --- a/task/yaml.go +++ b/task/yaml.go @@ -32,6 +32,8 @@ func GetYamlData() error { } // Synchronize other sources of yaml version taskhandler.SyncEulerYaml() + // Delete historical yaml source version data that does not exist + taskhandler.DelHistoryEulerYaml() logs.Info("End of the task of obtaining yaml data source") return err } diff --git a/taskhandler/assist.go b/taskhandler/assist.go index e41ad80..4ca3056 100644 --- a/taskhandler/assist.go +++ b/taskhandler/assist.go @@ -93,7 +93,6 @@ func GetBranchesInfo(accessToken, owner, path string) ([]string, error) { logs.Info(mapValue) if mapValue != "" && len(mapValue) > 3 { if mapValue == "openEuler-20.03-LTS" || - mapValue == "openEuler-20.09" || mapValue == "openEuler-20.03-LTS-SP1" { branchNameList = append(branchNameList, mapValue) } diff --git a/taskhandler/common.go b/taskhandler/common.go index ce1db89..95a92e2 100644 --- a/taskhandler/common.go +++ b/taskhandler/common.go @@ -237,7 +237,7 @@ func GitOpenEulerData(values map[string]interface{}, ge *models.GitOpenEuler, tb defer common.Catchs() ge.TableName = tb.TableName ge.TableId = tb.TableId - ge.Status = 0 + ge.Status = 1 if values["feature"] == nil { ge.Feature = "" } else { diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 11ebbb5..177b9cf 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -716,8 +716,10 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef string, // Synchronize the data returned by the Chinese Academy of Sciences func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) (bool, error) { if cveData.Ids == "" || cveData.CveNum == "" { - logs.Error("GenCveVuler, The current data cveNum is empty and will not be processed temporarily, data: ", cveData) - models.UpdateOriginStatus(common.GetCurTime(), cveData.PackName, cveData.Version, cveData.CveId, 3) + logs.Error("GenCveVuler, The current data cveNum is empty " + + "and will not be processed temporarily, data: ", cveData) + models.UpdateOriginStatus(common.GetCurTime(), + cveData.PackName, cveData.Version, cveData.CveId, 3) return false, errors.New("数据错误，暂时不处理") } BConfig, err := config.NewConfig("ini", "conf/app.conf") @@ -939,6 +941,7 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe var god models.GitOpenEuler god.PackageName = cveData.PackName god.Version = cveData.CveVersion + god.Status = 1 ok := models.QueryCveOpeneulerdata(&god) if !ok { if organizationID == 1 { diff --git a/taskhandler/grabissue.go b/taskhandler/grabissue.go index 83e1426..df07243 100644 --- a/taskhandler/grabissue.go +++ b/taskhandler/grabissue.go @@ -169,7 +169,7 @@ func handleIssueList(list []models.HookIssue, product, desc string) { if strings.HasPrefix(issueTitle, "CVE") { item.CveNumber = issueTitle } else if v.Body != "" { - //通过正则表达式去body中截取 稍后完善 + // Use regular expressions to intercept the body and improve it later sm := util.RegexpCveNumber.FindAllStringSubmatch(v.Body, -1) if len(sm) > 0 && len(sm[0]) > 0 { item.CveNumber = util.TrimString(sm[0][1]) diff --git a/taskhandler/oricvecheck.go b/taskhandler/oricvecheck.go index 7ceb984..e7a005e 100644 --- a/taskhandler/oricvecheck.go +++ b/taskhandler/oricvecheck.go @@ -42,6 +42,7 @@ func CheckCveOriginData(prcnum int) (string, error) { var gits models.GitOpenEuler gits.PackageName = pkList[0] gits.Version = pkList[1] + gits.Status = 1 ok := models.QueryCveOpeneulerdata(&gits) if !ok { opy := models.OpenGussYaml{PackageName: pkList[0], Version: pkList[1]} diff --git a/taskhandler/yaml.go b/taskhandler/yaml.go index 5bc832f..4c1e8ae 100644 --- a/taskhandler/yaml.go +++ b/taskhandler/yaml.go @@ -478,7 +478,7 @@ func UpdateYamlToDb(er models.OpenEulerRepoOrigin) { if len(oro.Decription) < 2 && len(er.Decription) > 2 { oro.Decription = er.Decription } - if len(er.Branchs) > 1 && !strings.Contains(oro.Branchs, er.Branchs){ + if len(er.Branchs) > 1 && !strings.Contains(oro.Branchs, er.Branchs) { oro.Branchs = oro.Branchs + "," + er.Branchs } if len(oro.Summary) < 2 && len(er.Summary) > 2 { @@ -508,24 +508,69 @@ func UpdateYamlToDb(er models.OpenEulerRepoOrigin) { func SyncEulerYaml() { // Query raw data yamlId := int64(0) - yamlList := models.GetEulerYamlList(yamlId, 100) - if len(yamlList) > 0 { - for _, yl := range yamlList { - if yl.PackageName == "" || yl.Version == "" { - logs.Error("yl: ", yl) - continue + for { + yamlList := models.GetEulerYamlList(yamlId, 100) + if len(yamlList) > 0 { + for _, yl := range yamlList { + if len(yl.PackageName) < 2 || len(yl.Version) < 1 { + logs.Error("yl: ", yl) + continue + } + yamlId = yl.Id + var ge models.GitOpenEuler + ge.PackageName = yl.PackageName + ge.Version = yl.Version + geErr := models.QueryYamlData(&ge, "PackageName", "Version") + if ge.GitId == 0 { + logs.Info("yaml data does not exist, need to insert data, err: ", + geErr, ",PackageName: ", yl.PackageName, ", Version: ", yl.Version) + AddYamlData(&ge, yl) + } else { + EditYamlData(&ge, yl) + } } - yamlId = yl.Id - var ge models.GitOpenEuler - ge.PackageName = yl.PackageName - ge.Version = yl.Version - geErr := models.QueryYamlData(&ge, "PackageName", "Version") - if ge.GitId == 0 { - logs.Info("yaml data does not exist, need to insert data, err: ", geErr) - AddYamlData(&ge, yl) - } else { - EditYamlData(&ge, yl) + } else { + break + } + } +} + +// Delete historical yaml source version data that does not exist +func DelHistoryEulerYaml() { + gitId := int64(0) + for { + yamlList := models.GetGitOpenEulerList(gitId, 100) + if len(yamlList) > 0 { + for _, yl := range yamlList { + if len(yl.PackageName) < 2 || len(yl.Version) < 1 { + logs.Error("yl: ", yl) + continue + } + gitId = yl.GitId + var oro models.OpenEulerRepoOrigin + oro.PackageName = yl.PackageName + oro.Version = yl.Version + repErr := models.QueryEulerRepoOrigin(&oro, "PackageName", "Version") + if oro.Id == 0 { + logs.Info("yaml data does not exist, delete data, err: ", + repErr, ",PackageName: ", yl.PackageName, ", Version: ", yl.Version) + yl.Status = 2 + upErr := models.UpdateYamlData(&yl, "Status") + if upErr == nil { + ge := models.GitPackageInfo{PackageName: yl.PackageName, Version: yl.Version} + qErr := models.QueryGitPackageInfo(&ge, "PackageName", "Version") + if qErr == nil { + ge.Status = 2 + upgErr := models.UpdateGitPackageInfo(&ge, "Status") + if upgErr != nil { + logs.Error("UpdateGitPackageInfo, upgErr: ", upgErr) + } + } + } + } } + } else { + break } } } diff --git a/util/time.go b/util/time.go index 531b00c..bf895fc 100644 --- a/util/time.go +++ b/util/time.go @@ -4,15 +4,16 @@ import ( "github.com/astaxie/beego/logs" "time" ) + //TimeStrToInt parse time string to unix nano -func TimeStrToInt(ts ,layout string) int64 { - if ts == ""{ +func TimeStrToInt(ts, layout string) int64 { + if ts == "" { return 0 } - if layout == ""{ + if layout == "" { layout = "2006-01-02 15:04:05" } - t, err := time.ParseInLocation(layout, ts,time.Local) + t, err := time.ParseInLocation(layout, ts, time.Local) if err != nil { logs.Error(err) return 0 -- Gitee