From c4146592b63b902c7a5926200209cdd6aa08d7d3 Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Wed, 28 Oct 2020 18:11:43 +0800 Subject: [PATCH 1/2] Rebuild the dockerfile environment of beego --- Dockerfile | 32 +++++++++---- conf/app.conf | 26 ++++++---- conf/product_app.conf | 8 ++-- controllers/file.go | 2 +- doc/sql/db_struct.sql | 55 +++++++++++---------- models/cve_web.go | 2 +- models/giteeissue.go | 10 ++++ models/issue.go | 10 ++-- models/modeldb.go | 2 +- models/uploadcve.go | 13 +++++ task/inittask.go | 2 - task/issuetask.go | 97 ++++++++++++++++++++------------------ taskhandler/common.go | 21 ++++++--- taskhandler/createissue.go | 8 ++-- taskhandler/cve.go | 89 ++++++++++++++++++++++------------ taskhandler/excel.go | 4 +- taskhandler/grabissue.go | 3 +- util/http.go | 12 ++--- 18 files changed, 246 insertions(+), 150 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d8809d..c9b62a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,30 @@ -FROM golang:latest as BUILDER +FROM golang:latest MAINTAINER TommyLike -# build binary -RUN mkdir -p /go/src/gitee.com/openeuler/cve-manager -COPY . /go/src/gitee.com/openeuler/cve-manager +WORKDIR $GOPATH/src/cve-manager +ADD . $GOPATH/src/cve-manager +#WORKDIR $GOPATH/src/arvonblog +COPY ./conf/product_app.conf $GOPATH/src/cve-manager/conf/app.conf +RUN go get github.com/astaxie/beedb && go get github.com/astaxie/beego && go get github.com/astaxie/beego/orm && go get github.com/astaxie/beego/toolbox + RUN cd /go/src/gitee.com/openeuler/cve-manager && CGO_ENABLED=1 go build -v -o ./cve-manager main.go +EXPOSE 8080 +ENTRYPOINT ["./cve-manager"] +#ENTRYPOINT ["./main"] + +# build binary +#RUN mkdir -p /go/src/gitee.com/openeuler/cve-manager +#COPY . /go/src/gitee.com/openeuler/cve-manager +#RUN cd /go/src/gitee.com/openeuler/cve-manager && CGO_ENABLED=1 go build -v -o ./cve-manager main.go + # copy binary config and utils -FROM golang:latest -RUN mkdir -p /opt/app/ && mkdir -p /opt/app/conf/ -COPY ./conf/product_app.conf /opt/app/conf/app.conf +#FROM golang:latest +#RUN mkdir -p /opt/app/ && mkdir -p /opt/app/conf/ +#COPY ./conf/product_app.conf /opt/app/conf/app.conf # overwrite config yaml -COPY --from=BUILDER /go/src/gitee.com/openeuler/cve-manager/cve-manager /opt/app +#COPY --from=BUILDER /go/src/gitee.com/openeuler/cve-manager/cve-manager /opt/app -WORKDIR /opt/app/ -ENTRYPOINT ["/opt/app/cve-manager"] \ No newline at end of file +#WORKDIR /opt/app/ +#ENTRYPOINT ["/opt/app/cve-manager"] \ No newline at end of file diff --git a/conf/app.conf b/conf/app.conf index fcc2323..38b1a43 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -19,13 +19,16 @@ rejectCmd = "/reject" approveCmd = "/approve" closeCmd = "/close" labelFixed = "CVE/FIXED" -labelUnFix = "CVE/UNFIX" - +labelUnFix = "CVE/UNFIXED" +fileDir = "download" +#release package download url +rpUrl = "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" [mysql] -dbhost = "${DB_URI||***}" + +dbhost = "${DB_URI||127.0.0.1}" dbport = 3306 -dbuser = "${DB_USER||cve}" +dbuser = "${DB_USER||root}" dbpwd = "${DB_PWD||***}" dbname = cvevulner dbprefix = cve_ @@ -42,9 +45,9 @@ maxsize=304800 [crontab] ymalflag = 2 -getymal = 00 00 01 * * * +getymal = 00 00 23 * * * cveflag = 2 -getcve = 00 00 03 * * * +getcve = 00 00 01 * * * oricveflag = 2 oricvecheck = 00 00 05 * * * getissueflag = 2 @@ -57,17 +60,24 @@ issueoath = * * */20 * * * genexcelflag = 7 genexcel = 00 00 04 * * * days = -30 -prcnum = 100 -printlogflag = 2 +prcnum = 50 +printlogflag = 1 printlog = 0 */10 * * * * [gitee] +#owner = cve-test +#owner = src-openeuler +#path = test +#email = 1499273991@qq.com +#redirect_uri = http://119.8.126.102:80/v1/issue/oauth/callback +# -------jianjun gitee 配置 -------- owner = cve-test path = jasper email = 7844966+zhangjianjun_code@user.noreply.gitee.com redirect_uri = http://159.138.2.2:80/v1/issue/oauth/callback + scope = user_info projects pull_requests issues notes keys hook groups gists enterprises emails # Obtain first from the system environment variables, use the default value if it fails **** client_id = "${GITEE_CLIENT_ID||****}" diff --git a/conf/product_app.conf b/conf/product_app.conf index c93cbc0..40a0b4f 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -19,8 +19,10 @@ rejectCmd = "/reject" approveCmd = "/approve" closeCmd = "/close" labelFixed = "CVE/FIXED" -labelUnFix = "CVE/UNFIX" - +labelUnFix = "CVE/UNFIXED" +fileDir = "download" +#release package download url +rpUrl = "http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" [mysql] #dbhost = 172.16.1.73 @@ -58,7 +60,7 @@ issueoath = * * */20 * * * genexcelflag = 2 genexcel = 00 00 04 * * * days = -30 -prcnum = 100 +prcnum = 50 printlogflag = 2 printlog = 0 */10 * * * * diff --git a/controllers/file.go b/controllers/file.go index 1024b15..a938e46 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -96,7 +96,7 @@ func (f *FileController) TriggerCveData() { if err != nil { f.Ctx.WriteString(fmt.Sprintf("error:%v", err)) } else { - //return the success notice to custom + //return the success notice f.Ctx.WriteString(fmt.Sprintf("Success：The name of the excel file generated this time is: %s. It takes some time to generate the excel file. "+ "You can try to call the download file interface and pass in the param fileCode=%s to be downloaded.", en, fileCode)) } diff --git a/doc/sql/db_struct.sql b/doc/sql/db_struct.sql index e1643b1..4f6404f 100644 --- a/doc/sql/db_struct.sql +++ b/doc/sql/db_struct.sql @@ -48,9 +48,11 @@ CREATE TABLE `cve_export_record` ( `id` bigint NOT NULL AUTO_INCREMENT, `file_name` varchar(255) NOT NULL DEFAULT '', `create_time` datetime NOT NULL, + `file_code` varchar(255) NOT NULL DEFAULT '', + `state` tinyint NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `file_name` (`file_name`) -) ENGINE=InnoDB AUTO_INCREMENT=15 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=16 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_open_euler` */ @@ -151,7 +153,7 @@ CREATE TABLE `cve_git_repo_groups` ( `group_name` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY (`group_id`), KEY `cve_git_repo_groups_group_name` (`group_name`) -) ENGINE=InnoDB AUTO_INCREMENT=144 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=146 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_sub_pack` */ @@ -260,7 +262,7 @@ CREATE TABLE `cve_gite_repo` ( `delete_time` varchar(255) DEFAULT NULL, PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_group_id` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=7096 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14269 DEFAULT CHARSET=utf8; /*Table structure for table `cve_gite_repo_branch` */ @@ -291,7 +293,7 @@ CREATE TABLE `cve_gite_repo_member` ( PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_member_group_id` (`group_id`), CONSTRAINT `member_groups_group_id` FOREIGN KEY (`group_id`) REFERENCES `cve_git_repo_groups` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=443 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=884 DEFAULT CHARSET=utf8; /*Table structure for table `cve_ip_white` */ @@ -332,7 +334,7 @@ CREATE TABLE `cve_issue_hooks` ( UNIQUE KEY `cve_issue_hooks_owner_repo_status_un` (`owner`,`repo`,`hook_url`,`status`), KEY `cve_issue_hooks_Cve_id` (`cve_id`), KEY `cve_issue_hooks_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=110 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=114 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_template` */ @@ -378,7 +380,7 @@ CREATE TABLE `cve_issue_template` ( UNIQUE KEY `cve_issue_template_cvn_issnu` (`cve_num`,`issue_num`), KEY `cve_issue_template_cve_id` (`cve_id`), KEY `cve_issue_template_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=482 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*Table structure for table `cve_open_euler_s_a` */ @@ -392,7 +394,7 @@ CREATE TABLE `cve_open_euler_s_a` ( PRIMARY KEY (`openeuler_id`), UNIQUE KEY `openeuler_sa_num` (`openeuler_sa_num`), KEY `cve_open_euler_s_a_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=546 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_excel` */ @@ -427,7 +429,7 @@ CREATE TABLE `cve_origin_excel` ( `delete_time` datetime DEFAULT NULL, PRIMARY KEY (`cve_id`) USING BTREE, UNIQUE KEY `cve_origin_excel_num_un` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=149 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; +) ENGINE=InnoDB AUTO_INCREMENT=152 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; /*Table structure for table `cve_origin_upstream` */ @@ -460,7 +462,7 @@ CREATE TABLE `cve_origin_upstream` ( KEY `cve_origin_upstream_cve_packname` (`cve_packname`), KEY `cve_origin_upstream_git_packname` (`git_packname`), KEY `cve_origin_upstream_version` (`version`) -) ENGINE=InnoDB AUTO_INCREMENT=43 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=1393 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config` */ @@ -472,7 +474,7 @@ CREATE TABLE `cve_origin_upstream_config` ( `nodes` varchar(32) DEFAULT NULL, PRIMARY KEY (`conf_id`), KEY `cve_origin_upstream_config_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2810 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node` */ @@ -484,7 +486,7 @@ CREATE TABLE `cve_origin_upstream_config_node` ( `operator` varchar(256) DEFAULT NULL, PRIMARY KEY (`node_id`), KEY `cve_origin_upstream_config_node_conf_id` (`conf_id`) -) ENGINE=InnoDB AUTO_INCREMENT=105 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=5303 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node_cpe` */ @@ -498,7 +500,7 @@ CREATE TABLE `cve_origin_upstream_config_node_cpe` ( `vulner_able` varchar(64) DEFAULT NULL, PRIMARY KEY (`cpe_id`), KEY `cve_origin_upstream_config_node_cpe_node_id` (`node_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2818 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=163418 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_desc` */ @@ -511,7 +513,7 @@ CREATE TABLE `cve_origin_upstream_desc` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`desc_id`), KEY `cve_origin_upstream_desc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2810 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_event` */ @@ -526,7 +528,7 @@ CREATE TABLE `cve_origin_upstream_event` ( `description` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`event_id`), KEY `cve_origin_upstream_event_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2810 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest` */ @@ -538,7 +540,7 @@ CREATE TABLE `cve_origin_upstream_fix_suggest` ( `detail` varchar(1024) DEFAULT NULL, PRIMARY KEY (`fix_id`), KEY `cve_origin_upstream_fix_suggest_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2798 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest_ref` */ @@ -576,7 +578,7 @@ CREATE TABLE `cve_origin_upstream_impact` ( `impact` varchar(32) DEFAULT NULL, PRIMARY KEY (`impact_id`), KEY `cve_origin_upstream_impact_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2810 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score` */ @@ -592,7 +594,7 @@ CREATE TABLE `cve_origin_upstream_impact_score` ( `score_status` tinyint DEFAULT '1', PRIMARY KEY (`score_id`), KEY `cve_origin_upstream_impact_score_impact_id` (`impact_id`) -) ENGINE=InnoDB AUTO_INCREMENT=167 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=5619 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v2` */ @@ -621,7 +623,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v2` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v2_id`), KEY `cve_origin_upstream_impact_score_v2_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2810 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v3` */ @@ -647,7 +649,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v3` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v3_id`), KEY `cve_origin_upstream_impact_score_v3_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2810 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_poc` */ @@ -664,7 +666,7 @@ CREATE TABLE `cve_origin_upstream_poc` ( `desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`poc_id`), KEY `cve_origin_upstream_poc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2810 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_reference` */ @@ -679,7 +681,7 @@ CREATE TABLE `cve_origin_upstream_reference` ( `tags` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`ref_id`), KEY `cve_origin_upstream_reference_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=1405 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=55467 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_vul_type` */ @@ -693,7 +695,7 @@ CREATE TABLE `cve_origin_upstream_vul_type` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`vul_id`), KEY `cve_origin_upstream_vul_type_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=84 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2798 DEFAULT CHARSET=utf8; /*Table structure for table `cve_other_user` */ @@ -781,7 +783,7 @@ CREATE TABLE `cve_score` ( PRIMARY KEY (`id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_score_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=546 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*Table structure for table `cve_score_record` */ @@ -796,7 +798,7 @@ CREATE TABLE `cve_score_record` ( `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `cve_score_record_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=546 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_notice` */ @@ -822,7 +824,7 @@ CREATE TABLE `cve_security_notice` ( PRIMARY KEY (`sec_id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_security_notice_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=547 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_reviewer` */ @@ -856,8 +858,9 @@ CREATE TABLE `cve_vuln_center` ( `data_source` tinyint NOT NULL DEFAULT '1', `cve_detail_url` varchar(1024) NOT NULL DEFAULT '', PRIMARY KEY (`cve_id`), + UNIQUE KEY `cve_vuln_center_num_pack_v_un` (`cve_num`,`cve_version`,`pack_name`), KEY `cve_vuln_center_cve_num` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=603 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB DEFAULT CHARSET=utf8; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; diff --git a/models/cve_web.go b/models/cve_web.go index b159029..150e7dc 100644 --- a/models/cve_web.go +++ b/models/cve_web.go @@ -7,7 +7,7 @@ type RespCommon struct { } type CveDetail struct { - Id int64 + Id int64 `json:"id"` CveId string `json:"cveId"` AffectedProduct string `json:"affectedProduct"` } diff --git a/models/giteeissue.go b/models/giteeissue.go index 0c2d0c6..72b4199 100644 --- a/models/giteeissue.go +++ b/models/giteeissue.go @@ -244,6 +244,16 @@ func parseOldTplToLoopHole(lp *Loophole, body string) { if len(sm) > 0 { lp.InfPackage = util.TrimString(sm[0][1]) } + if lp.OpVector == "" { + if lp.OpScore == "" { + lp.OpScore = lp.CvsScore + lp.OpVector = lp.CvsVector + }else { + if lp.OpScore == lp.CvsScore { + lp.OpVector = lp.CvsVector + } + } + } } func parseNewTplToLoopHole(lp *Loophole, body string) { diff --git a/models/issue.go b/models/issue.go index 7e78f58..393a6a2 100644 --- a/models/issue.go +++ b/models/issue.go @@ -19,7 +19,7 @@ func QueryIssueCveByNum(cvenum, repoPath string) (GiteOriginIssue, bool) { var cve GiteOriginIssue err := o.Raw("select * from cve_gite_origin_issue where cve_number = ? and repo_path = ?", cvenum, repoPath).QueryRow(&cve) if err != nil { - logs.Error(cvenum, "cve_gite_origin_issue 查询不到") + logs.Info(cvenum, ", cve_gite_origin_issue 查询不到, 不是错误,需要新增数据") return cve, false } else { return cve, true @@ -199,9 +199,11 @@ func ReplacePackageByCveId(pkgList []string,cveId int64) error { if strings.Trim(v," ")==""{ continue } - platform := "aarch64" + platform := "source" if strings.Contains(v,".x86_64."){ platform = "x86_64" + } else if strings.Contains(v,".aarch64.") || strings.Contains(v,".aarch64."){ + platform = "aarch64" } pkgUrl := fmt.Sprintf(`https://repo.openeuler.org/openEuler-20.03-LTS/update/%s/Packages/%s`,platform, v) pv := Package{SecId: sec.SecId, PackName: v, PackUrl: pkgUrl} @@ -223,8 +225,8 @@ func CreateIssueTemplate(it *IssueTemplate) (issTempId int64, err error) { o := orm.NewOrm() var localIt IssueTemplate errx := o.Raw("select *"+ - " from cve_issue_template where cve_id = ? ", it.CveId).QueryRow(&localIt) - if errx != nil { + " from cve_issue_template where cve_num = ? and issue_num = ?", it.CveNum, it.IssueNum).QueryRow(&localIt) + if errx != nil || localIt.IssueId == 0{ var issTempId int64 if issTempId, err = o.Insert(it); err == nil { logs.Info("insert cve_issue_template success, issTempId: ", issTempId, "cveNum: ", it.CveNum) diff --git a/models/modeldb.go b/models/modeldb.go index 59bc307..92954c2 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -38,7 +38,7 @@ type VulnCenter struct { CveNum string `orm:"size(256);column(cve_num);index" description:"cve编号"` Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` - Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue"` + Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败"` CveVersion string `orm:"size(128);column(cve_version)" description:"cve归属版本"` RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` diff --git a/models/uploadcve.go b/models/uploadcve.go index 4ee45a2..25ce827 100644 --- a/models/uploadcve.go +++ b/models/uploadcve.go @@ -102,6 +102,19 @@ func QueryCveOpeneulerDetaildata(pkgName string, version string) (GitPackageInfo } } +func QueryCveOpeneulerDetaildataByName(pkgName string) (GitPackageInfo, bool) { + o := orm.NewOrm() + var goe GitPackageInfo + err := o.Raw("select * from cve_git_package_info where package_name = ? order by detail_id limit 1", pkgName).QueryRow(&goe) + if err != nil { + logs.Info("pkgName: ", pkgName, ", cve_git_package_info 无对应数据") + return goe, false + } else { + logs.Info("pkgName: ", pkgName, ", cve_git_package_info 查询成功") + return goe, true + } +} + func QueryCveOriginByIds(ids string) (OriginUpstream, bool) { o := orm.NewOrm() var orcve OriginUpstream diff --git a/task/inittask.go b/task/inittask.go index 19a0cc1..362fcb9 100644 --- a/task/inittask.go +++ b/task/inittask.go @@ -171,8 +171,6 @@ func InitTask() bool { gittokenflag, errx := BConfig.Int("crontab::gittokenflag") if gittokenflag == 1 && errx == nil { issueoath := BConfig.String("crontab::issueoath") - // Refresh token regularly - //go InitIssueToken(issueoath, &ch) // Get token from environment variable gitToken := BConfig.String("gitee::git_token") if gitToken == "xxx" { diff --git a/task/issuetask.go b/task/issuetask.go index 62e5ff5..3832a61 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -69,63 +69,65 @@ func CreateIssue() error { } days, ok := BConfig.Int("crontab::days") if ok != nil { - logs.Error("config crontab::days error:", err) + logs.Error("config crontab::days error:", ok) return ok } prcnum, ok := BConfig.Int("crontab::prcnum") if ok != nil { - logs.Error("config crontab::prcnum error:", err) + logs.Error("config crontab::prcnum error:", ok) return ok } beforeTime := common.GetBeforeTime(days) - cveData, err := models.QueryIssue(beforeTime, prcnum) - if err == nil && len(cveData) > 0 { - logs.Info(cveData) - } else { - logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime()) - return err - } - accessToken := os.Getenv("GITEE_TOKEN") - if accessToken == "" || len(accessToken) < 1 { - logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) - return err - } - owner := BConfig.String("gitee::owner") - path := BConfig.String("gitee::path") - for index, issueValue := range cveData { - logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) - // Determine whether the issue has been processed - goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) - if oks { - if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) - continue - } + for ; ;{ + cveData, err := models.QueryIssue(beforeTime, prcnum) + if err == nil && len(cveData) > 0 { + logs.Info(cveData) + } else { + logs.Info("无cve数据可以使用, 当前时间: ", common.GetCurTime(), ", err: ", err) + return err } - // Determine whether cve has been processed - issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) - if issueExist { - models.UpdateIssueStatus(issueValue, 2) - logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) - continue + accessToken := os.Getenv("GITEE_TOKEN") + if accessToken == "" || len(accessToken) < 1 { + logs.Error("issue token 获取失败, 当前时间: ", common.GetCurTime()) + return err } - // Process each piece of cve data - if issueValue.Status == 0 { - err := ProcIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) - continue + owner := BConfig.String("gitee::owner") + path := BConfig.String("gitee::path") + for index, issueValue := range cveData { + logs.Info("当前正在处理第：", index, "条cve数据, cveNum: ", issueValue.CveNum) + // Determine whether the issue has been processed + goi, oks := models.QueryIssueCveByNum(issueValue.CveNum, issueValue.PackName) + if oks { + if strings.ToLower(goi.State) == "closed" || strings.ToLower(goi.State) == "rejected" || + goi.State == "已完成" || goi.State == "已拒绝"{ + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) + continue + } } - } else { - err := ProcUpdateIssue(issueValue, accessToken, owner, path) - if err != nil { - logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + // Determine whether cve has been processed + issueExist := taskhandler.GetCveSecurityNotice(issueValue.CveNum) + if issueExist { + models.UpdateIssueStatus(issueValue, 2) + logs.Info("cve数据已经在官网上展示过,不需要在git上提交issue, cveData: ", issueValue) continue } + // Process each piece of cve data + if issueValue.Status == 0 { + err := ProcIssue(issueValue, accessToken, owner, path) + if err != nil { + logs.Error("创建issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } else { + err := ProcUpdateIssue(issueValue, accessToken, owner, path) + if err != nil { + logs.Error("修改issue失败, cvenum: ", issueValue.CveNum, "err,err: ", err) + continue + } + } } } - return nil } //ProcUpdateIssue Update issue @@ -150,6 +152,8 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner, path stri issueValue, lit) if err != nil { logs.Error("更新issue 模板失败， cveId: ", issueValue.CveId, "err: ", err) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) return err } // Update issue status @@ -180,8 +184,9 @@ func ProcIssue(issueValue models.VulnCenter, accessToken, owner, path string) er if !ok || gitYaml.MainTainer == "" || len(gitYaml.MainTainer) < 1 { assignee, err = taskhandler.GetCollaboratorInfo(accessToken, owner, path) if assignee == "" { - logs.Error("获取仓库： owner:", owner, "path:", path, "分析人失败", "err:", err, "cveid: ", issueValue.CveId) - return err + logs.Error("获取仓库： owner:", owner, "path:", path, "分析人失败", "err:", err, + "cveid: ", issueValue.CveId, ",创建无maintainer的issue") + //return err } } else { assignee = gitYaml.MainTainer @@ -234,6 +239,8 @@ func ProcIssue(issueValue models.VulnCenter, accessToken, owner, path string) er resp, err := taskhandler.CreateIssueToGit(accessToken, owner, path, assignee, issueValue, sc, brandArry) if err != nil { logs.Error("创建issue失败, err: ", err, "resp: ", resp, "cveId: ", issueValue.CveId) + // Update issue status + models.UpdateIssueStatus(issueValue, 3) return err } // Store security bulletin related information diff --git a/taskhandler/common.go b/taskhandler/common.go index 0da4436..5c72974 100644 --- a/taskhandler/common.go +++ b/taskhandler/common.go @@ -307,6 +307,9 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str } else { scoreType = "3.0" } + cve.Description = strings.ReplaceAll(cve.Description, "\"", "") + cve.Description = strings.ReplaceAll(cve.Description, "'", "") + cve.Description = strings.ReplaceAll(cve.Description, "\\", "") BConfig, err := config.NewConfig("ini", "conf/app.conf") if err != nil { logs.Error("config init error:", err) @@ -331,10 +334,8 @@ func CreateIssueBody(accessToken string, owner string, path string, assignee str StatusName = "open" } else if its.StatusName == "进行中" || strings.ToLower(its.StatusName) == "started" { StatusName = "progressing" - } else if its.StatusName == "已完成" { + } else if its.StatusName == "已完成" || its.StatusName == "已拒绝"{ StatusName = "closed" - } else if its.StatusName == "已拒绝" { - StatusName = "rejected" } else { StatusName = its.StatusName } @@ -523,9 +524,15 @@ func Duplicate(a interface{}) (ret []interface{}) { return ret } -func RemoveSubstring(s, sub string) string { - if strings.Contains(s, sub) { - return strings.ReplaceAll(s, sub, "") +func RemoveSubstring(s string, subList []string) string { + newStr := s + if len(subList) > 0 { + for _, sub := range subList { + if strings.Contains(newStr, sub) { + newStr = strings.ReplaceAll(newStr, sub, "") + } + } } - return s + logs.Info("s: ",s, ",newstr: ", newStr) + return newStr } diff --git a/taskhandler/createissue.go b/taskhandler/createissue.go index 1c1df10..6b492e5 100644 --- a/taskhandler/createissue.go +++ b/taskhandler/createissue.go @@ -115,7 +115,7 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st } } else { issueType := "CVE和安全问题" - labels := beego.AppConfig.String("labelUnFix") + labels := beego.AppConfig.String("labelFixed") if accessToken != "" && owner != "" && path != "" { url := "https://gitee.com/api/v5/repos/" + owner + "/issues" score := strconv.FormatFloat(sc.NVDScore, 'f', 1, 64) @@ -124,7 +124,7 @@ func CreateIssueToGit(accessToken string, owner string, path string, assignee st logs.Info("isssue_body: ", requestBody) resp, err := util.HTTPPost(url, requestBody) if err != nil { - logs.Error("创建issue失败, cveNum: ", cve.CveNum, "err: ", err) + logs.Error("url: ", url, "创建issue失败, cveNum: ", cve.CveNum, "err: ", err) return "", err } if _, ok := resp["id"]; !ok { @@ -483,7 +483,7 @@ func CreateSecNoticeData(sec *models.SecurityNotice, iss models.VulnCenter, path } sec.CveId = iss.CveId sec.CveNum = iss.CveNum - sec.Introduction = "An update for " + path + " is now available for " + branchs + sec.Introduction = "An update for " + path + " is now available for " + branchs + "." sec.Theme = sec.Introduction + ";\n\n" + "openEuler Security has rated this" + " update as having a security impact of medium. A Common Vunlnerability" + " Scoring System(CVSS)base score,which gives a detailed severity rating," + @@ -497,7 +497,7 @@ func CreateIssueLabel(accessToken string, owner string, path string, url := "https://gitee.com/api/v5/repos/" + owner + "/" + path + "/issues/" + issNum + "/labels" body := make(map[string]interface{}) body["access_token"] = accessToken - body["body"] = fmt.Sprintf(`[%s,%s]`,beego.AppConfig.String("labelUnFix"),beego.AppConfig.String("labelFixed")) + body["body"] = fmt.Sprintf(`[%s,%s]`, beego.AppConfig.String("labelUnFix"), beego.AppConfig.String("labelFixed")) requestBody, _ := json.Marshal(body) logs.Info("create issue label: ", string(requestBody)) resp, err := util.HTTPPost1(url, string(requestBody)) diff --git a/taskhandler/cve.go b/taskhandler/cve.go index 38127fa..fd9fe6c 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -12,6 +12,7 @@ import ( "io" "io/ioutil" "net/http" + "net/url" "os" "path/filepath" "strconv" @@ -20,7 +21,9 @@ import ( "time" ) -var GetCveDetailUrl = "http://cve.openeuler.org/cve-security-notice-server/cvedatabase/getByCveId?cveId=%s" + +var GetCveDetailUrl = "https://openeuler.org/api-cve/cve-security-notice-server/cvedatabase/getByCveId?cveId=%s" + var ( pkgLock sync.Mutex lockx sync.Mutex @@ -165,7 +168,7 @@ func UpdateExcelCveGroups(cveData models.OriginExcel, cveRef string, openeulerNu sec.OpenId = OpenEulId sec.InfluenceComponent = cveData.PackName sec.Status = 0 - sec.AffectStatus = "UnFixed" + sec.AffectStatus = "Fixed" sec.CveId = CveRes.CveId sec.Summary = cveData.PackName + " security update" sec.ReferenceLink = cveRef + cveData.CveNum @@ -220,13 +223,14 @@ func InsertCveExcelGroups(cveData models.OriginExcel, cveRef string, openeulerNu sec.CveNum = cveData.CveNum sec.InfluenceComponent = cveData.PackName sec.Status = 0 - sec.AffectStatus = "UnFixed" + sec.AffectStatus = "Fixed" sec.Summary = cveData.PackName + " security update" sec.ReferenceLink = cveRef + cveData.CveNum + specCharList := []string{"\n"} if goe.Decription != "" && len(goe.Decription) > 0 { - sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + cveData.CveDesc + "(" + vul.CveNum + ")" + sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + RemoveSubstring(cveData.CveDesc, specCharList) + "(" + vul.CveNum + ")" } else { - sec.Description = "Security Fix(es):" + "\n\n" + cveData.CveDesc + "(" + vul.CveNum + ")" + sec.Description = "Security Fix(es):" + "\n\n" + RemoveSubstring(cveData.CveDesc, specCharList) + "(" + vul.CveNum + ")" } var sc models.Score sc.CveNum = cveData.CveNum @@ -486,6 +490,7 @@ func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum } SecNOtice, secerrx := models.QuerySecNoticeByCveId(CveRes.CveId) if secerrx { + SecNOtice.CveId = CveRes.CveId SecNOtice.InfluenceComponent = pkList[0] SecNOtice.OpenId = OpenEulId SecNOtice.Summary = pkList[0] + " security update" @@ -496,7 +501,7 @@ func UpdateCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum sec.OpenId = OpenEulId sec.InfluenceComponent = pkList[0] sec.Status = 0 - sec.AffectStatus = "UnFixed" + sec.AffectStatus = "Fixed" sec.CveId = CveRes.CveId sec.Summary = pkList[0] + " security update" sec.ReferenceLink = cveRef + cveData.CveNum @@ -553,13 +558,14 @@ func InsertCveGroups(cveData models.OriginUpstream, cveRef string, openeulerNum sec.CveNum = cveData.CveNum sec.InfluenceComponent = pkList[0] sec.Status = 0 - sec.AffectStatus = "UnFixed" + sec.AffectStatus = "Fixed" sec.Summary = pkList[0] + " security update" sec.ReferenceLink = cveRef + cveData.CveNum + specCharList := []string{"\n"} if goe.Decription != "" && len(goe.Decription) > 0 { - sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + cveDesc.EnDescription + "(" + vul.CveNum + ")" + sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + RemoveSubstring(cveDesc.EnDescription, specCharList) + "(" + vul.CveNum + ")" } else { - sec.Description = "Security Fix(es):" + "\n\n" + cveDesc.EnDescription + "(" + vul.CveNum + ")" + sec.Description = "Security Fix(es):" + "\n\n" + RemoveSubstring(cveDesc.EnDescription, specCharList) + "(" + vul.CveNum + ")" } var sc models.Score sc.CveNum = cveData.CveNum @@ -731,7 +737,7 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) if len(versionList) > 0 { for _, ver := range versionList { if ver != "" && len(ver) > 1 { - goe, ok = models.QueryCveOpeneulerDetaildata(key, pkList[1]) + goe, ok = models.QueryCveOpeneulerDetaildataByName(key) if ok { versionFlag = true break @@ -792,7 +798,7 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum int) ( } else { logs.Info("获取到的数据: god:", god) } - goe, ok := models.QueryCveOpeneulerDetaildata(cveData.PackName, cveData.CveVersion) + goe, ok := models.QueryCveOpeneulerDetaildataByName(cveData.PackName) if !ok { logs.Error("仓库存在，yaml数据源不存在,god: ", god, ", cveData: ", cveData) } @@ -822,7 +828,6 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum int) ( func GetCveOriginData(prcnum, days, openeulernum int, cveRef string) (bool, error) { defer common.Catchs() - //var os []models.OriginUpstream count := 0 beforeTime := common.GetBeforeTime(days) for { @@ -892,9 +897,10 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c cveStatus int8, goe models.GitPackageInfo, owner string) (bool, error) { var vul models.VulnCenter vul.CveNum = cveData.CveNumber - vul.Description = RemoveSubstring(lop.BriefIntroduction, "

") + specCharList := []string{"

", "\n"} + vul.Description = RemoveSubstring(lop.BriefIntroduction, specCharList) vul.Status = cveStatus - vul.CveVersion = RemoveSubstring(lop.Version, "

") + vul.CveVersion = RemoveSubstring(lop.Version, specCharList) vul.RepairTime = "" vul.PackName = cveData.RepoPath vul.CveUrl = cveRef + cveData.CveNumber @@ -917,11 +923,12 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c sec.CveNum = cveData.CveNumber sec.InfluenceComponent = lop.Components sec.Status = 0 - sec.AffectStatus = "UnFixed" + sec.AffectStatus = "Fixed" sec.Summary = lop.Components + " security update" - sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + RemoveSubstring(lop.BriefIntroduction + "(" + vul.CveNum + ")", "

") + sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + + RemoveSubstring(lop.BriefIntroduction, specCharList) + "(" + vul.CveNum + ")" sec.ReferenceLink = cveRef + cveData.CveNumber - sec.Introduction = "An update for " + lop.Components + " is now available for " + lop.InfProduct + sec.Introduction = "An update for " + lop.Components + " is now available for " + lop.InfProduct + "." sec.Theme = sec.Introduction + ";\n\n" + "openEuler Security has rated this" + " update as having a security impact of medium. A Common Vunlnerability" + " Scoring System(CVSS)base score,which gives a detailed severity rating," + @@ -998,12 +1005,12 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c issueTemp.CveId = cveid issueTemp.CveNum = cveData.CveNumber issueTemp.OwnedComponent = lop.Components - issueTemp.OwnedVersion = RemoveSubstring(lop.Version, "

") + issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) issueTemp.NVDScore = nVDScore issueTemp.OpenEulerScore = openEulerScore issueTemp.NVDVector = lop.CvsVector issueTemp.OpenEulerVector = lop.OpVector - issueTemp.CveBrief = RemoveSubstring(lop.BriefIntroduction, "

") + issueTemp.CveBrief = RemoveSubstring(lop.BriefIntroduction, specCharList) issueTemp.CveAnalysis = lop.Influences issueTemp.PrincipleAnalysis = lop.Principle issueTemp.AffectedVersion = lop.InfVersion @@ -1052,23 +1059,26 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c cveStatus int8, goe models.GitPackageInfo, vul models.VulnCenter, owner string) (bool, error) { //var vul models.VulnCenter vul.CveNum = cveData.CveNumber - vul.Description = RemoveSubstring(lop.BriefIntroduction, "

") + specCharList := []string{"

", "\n"} + vul.Description = RemoveSubstring(lop.BriefIntroduction, specCharList) vul.Status = cveStatus - vul.CveVersion = RemoveSubstring(lop.Version, "

") + vul.CveVersion = RemoveSubstring(lop.Version, specCharList) vul.RepairTime = "" vul.PackName = cveData.RepoPath vul.CveUrl = cveRef + cveData.CveNumber vul.IsExport = 0 vul.DataSource = 4 var sec models.SecurityNotice + sec.CveId = vul.CveId sec.CveNum = cveData.CveNumber sec.InfluenceComponent = lop.Components sec.Status = 0 sec.AffectStatus = "Fixed" sec.Summary = lop.Components + " security update" - sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + RemoveSubstring(lop.BriefIntroduction + "(" + vul.CveNum + ")", "

") + sec.Description = goe.Decription + "\n\n" + "Security Fix(es):" + "\n\n" + + RemoveSubstring(lop.BriefIntroduction, specCharList) + "(" + vul.CveNum + ")" sec.ReferenceLink = cveRef + cveData.CveNumber - sec.Introduction = "An update for " + lop.Components + " is now available for " + lop.InfProduct + sec.Introduction = "An update for " + lop.Components + " is now available for " + lop.InfProduct + "." sec.Theme = sec.Introduction + ";\n\n" + "openEuler Security has rated this" + " update as having a security impact of medium. A Common Vunlnerability" + " Scoring System(CVSS)base score,which gives a detailed severity rating," + @@ -1077,6 +1087,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c nVDScore := 0.0 //var err error var score models.Score + score.CveId = vul.CveId var scOk bool score, scOk = models.QueryScoreByCveId(vul.CveId) if !scOk || score.Id == 0 { @@ -1223,6 +1234,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c scorecode.NVDScore = score.NVDScore scorecode.NvectorVule = score.NvectorVule scorecode.Status = 0 + scorecode.CveId = vul.CveId scRecordId, scError := models.InsertScoreRecord(&scorecode) logs.Info(scRecordId, scError) } @@ -1233,13 +1245,13 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c if okl && localt.TemplateId > 0 { issueTemp.CveNum = cveData.CveNumber issueTemp.OwnedComponent = lop.Components - issueTemp.OwnedVersion = RemoveSubstring(lop.Version, "

") + issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) if openError == nil { issueTemp.OpenEulerScore = openEulerScore } issueTemp.OpenEulerVector = lop.OpVector - issueTemp.CveBrief = RemoveSubstring(lop.BriefIntroduction, "

") + issueTemp.CveBrief = RemoveSubstring(lop.BriefIntroduction, specCharList) issueTemp.CveAnalysis = lop.Influences issueTemp.PrincipleAnalysis = lop.Principle issueTemp.AffectedVersion = lop.InfVersion @@ -1271,7 +1283,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c } else { issueTemp.CveNum = cveData.CveNumber issueTemp.OwnedComponent = lop.Components - issueTemp.OwnedVersion = RemoveSubstring(lop.Version, "

") + issueTemp.OwnedVersion = RemoveSubstring(lop.Version, specCharList) issueTemp.NVDScore = nVDScore openEulerScore, openError := strconv.ParseFloat(lop.OpScore, 64) if openError == nil { @@ -1279,7 +1291,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c } issueTemp.NVDVector = lop.CvsVector issueTemp.OpenEulerVector = lop.OpVector - issueTemp.CveBrief = RemoveSubstring(lop.BriefIntroduction, "

") + issueTemp.CveBrief = RemoveSubstring(lop.BriefIntroduction, specCharList) issueTemp.CveAnalysis = lop.Influences issueTemp.PrincipleAnalysis = lop.Principle issueTemp.AffectedVersion = lop.InfVersion @@ -1330,7 +1342,7 @@ func GenCveVulerByIssue(cveData models.GiteOriginIssue, cveRef string, openeuler models.UpdateCveIssueStatusById(3, cveData.Id) return false, err } - goe, ok := models.QueryCveOpeneulerDetaildata(cveData.RepoPath, hole.Version) + goe, ok := models.QueryCveOpeneulerDetaildataByName(cveData.RepoPath) if !ok || goe.DetailId == 0 { logs.Error("获取数据失败，", "Components: ", hole.Components, ",Version: ", hole.Version) } @@ -1420,6 +1432,7 @@ func GetCveIssueData(prcnum, days, openeulernum int, cveRef, owner string) (bool ok, err := GenCveVulerByIssue(cveData, cveRef, openeulernum, owner) if !ok { logs.Error("cveData: ", cveData, "处理失败, err: ", err) + models.UpdateCveIssueStatusById(3, cveOrg.Id) } ch <- idx }(i, cveOrg) @@ -1435,17 +1448,33 @@ func GetCveIssueData(prcnum, days, openeulernum int, cveRef, owner string) (bool //GetSecurityNotice Go to the CVE official website to obtain the cve data to determine //whether the cve issue needs to be exported. func GetCveSecurityNotice(cveNumber string) bool { - resp, err := http.Get(fmt.Sprintf(GetCveDetailUrl, cveNumber)) + var urlS url.URL + q := urlS.Query() + q.Add("cveId", cveNumber) + params := q.Encode() + req, _ := http.NewRequest("GET", "https://openeuler.org", nil) + req.URL = &url.URL{ + Scheme: req.URL.Scheme, + Host: req.URL.Host, + RawQuery: params, + Path: "/api-cve/cve-security-notice-server/cvedatabase/getByCveId", + } + + req.URL.EscapedPath() + //body, err := util.HTTPGetCom(req.URL.String()) + //resp, err := http.Get(fmt.Sprintf(GetCveDetailUrl, cveNumber)) + resp, err := http.Get(req.URL.String()) if err != nil { logs.Error(err) return false } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) - if err != nil { + if err != nil || body == nil{ logs.Error(err) return false } + logs.Info("url: ", req.URL.String(),"获取官网数据： ", body) var detail models.RespCveDetail err = json.Unmarshal(body, &detail) if err != nil { diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 484b889..1f5ef75 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -550,7 +550,7 @@ func (ec *CveExcel) setContentRow(v models.ExcelExport) { rCN := "B" + row rRl := "J" + row vcn, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rCN) - vcn += "\n" + v.CveNum + vcn += ";\n" + v.CveNum _ = ec.ExcelHandel.SetCellValue(ec.SecNoticeSheetName, rCN, vcn) vrl, _ := ec.ExcelHandel.GetCellValue(ec.SecNoticeSheetName, rRl) vrl += "\n" + v.ReferenceLink @@ -785,6 +785,7 @@ func getRepoAllPR(token, owner, repo string, startTime, releaseTime int64) (prLi q.Add("sort", "created") q.Add("state", "merged") q.Add("per_page", strconv.Itoa(pageSize)) + q.Add("base","openEuler-20.03-LTS") //target branch is openEuler-20.03-LTS for { q.Del("page") q.Add("page", strconv.Itoa(pageCount)) @@ -808,6 +809,7 @@ func getRepoAllPR(token, owner, repo string, startTime, releaseTime int64) (prLi break } for _, v := range pr { + ct := v.MergedAt.Local().Unix() if ct >= startTime && ct <= releaseTime { prList = append(prList, v) diff --git a/taskhandler/grabissue.go b/taskhandler/grabissue.go index 8bcc0ce..8583e91 100644 --- a/taskhandler/grabissue.go +++ b/taskhandler/grabissue.go @@ -129,7 +129,7 @@ func getInfProduct(token string, owner string, repo string) (infPro string, err return "", err } for _, v := range branchList { - if strings.HasSuffix(v.Name, "-LTS") { + if strings.HasSuffix(v.Name, "-20.03-LTS") { infPro = v.Name break } @@ -226,6 +226,7 @@ func GetIssueList(accToken, owner, repo, state string, page int) (issueList []mo if err != nil { return issueList, err } + logs.Error(string(body)) err = json.Unmarshal(body, &issueList) return } diff --git a/util/http.go b/util/http.go index a931750..70007aa 100644 --- a/util/http.go +++ b/util/http.go @@ -42,7 +42,7 @@ func HTTPPatch(url string, requestBody string) (map[string]interface{}, error) { } body, err := ioutil.ReadAll(resp.Body) fmt.Println("response Body:", string(body)) - if err != nil { + if err != nil || body == nil{ logs.Error("post 返回失败, err: ", err, "body: ", requestBody) return nil, err } @@ -80,7 +80,7 @@ func HTTPPost(url string, requestBody string) (map[string]interface{}, error) { } body, err := ioutil.ReadAll(resp.Body) fmt.Println("response Body:", string(body)) - if err != nil { + if err != nil || body == nil { logs.Error("post 返回失败, err: ", err, "body: ", requestBody) return nil, err } @@ -118,7 +118,7 @@ func HTTPPost1(url string, requestBody string) ([]map[string]interface{}, error) } body, err := ioutil.ReadAll(resp.Body) fmt.Println("response Body:", string(body)) - if err != nil { + if err != nil || body == nil{ logs.Error("post 返回失败, err: ", err, "body: ", requestBody) return nil, err } @@ -159,7 +159,7 @@ func PostURLEncoded(this RequestInfo) ([]byte, error) { defer resp.Body.Close() //Read return value result, err := ioutil.ReadAll(resp.Body) - if err != nil { + if err != nil || result == nil{ logs.Error(err) return nil, err } @@ -176,7 +176,7 @@ func HTTPGet(url string) ([]map[string]interface{}, error) { } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) - if err != nil { + if err != nil || body == nil { logs.Error("url:", url, ",err: ", err) return nil, err } @@ -199,7 +199,7 @@ func HTTPGetCom(urls string) ([]byte, error) { } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) - if err != nil { + if err != nil || body == nil{ logs.Error(err) return nil, err } -- Gitee From 3945a760f46ee80784e2b4ca3ce794291702d47f Mon Sep 17 00:00:00 2001 From: zhangjianjun_code <7844966+zhangjianjun_code@user.noreply.gitee.com> Date: Wed, 28 Oct 2020 18:23:15 +0800 Subject: [PATCH 2/2] Rebuild the dockerfile environment of beego --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c9b62a3..0119b06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,7 +8,7 @@ ADD . $GOPATH/src/cve-manager COPY ./conf/product_app.conf $GOPATH/src/cve-manager/conf/app.conf RUN go get github.com/astaxie/beedb && go get github.com/astaxie/beego && go get github.com/astaxie/beego/orm && go get github.com/astaxie/beego/toolbox -RUN cd /go/src/gitee.com/openeuler/cve-manager && CGO_ENABLED=1 go build -v -o ./cve-manager main.go +RUN go build -v -o ./cve-manager main.go EXPOSE 8080 ENTRYPOINT ["./cve-manager"] -- Gitee