diff --git a/conf/app.conf b/conf/app.conf index f06b5dff12884f81fb84ba512f0331149caee7a1..a19a796cbef934c80ea9f3efdc24f659211dbf64 100644 --- a/conf/app.conf +++ b/conf/app.conf @@ -38,7 +38,7 @@ maxconn = 3000 [log] -log_level = 5 +log_level = 7 log_dir = ./logs log_path = logs/cve.log maxlines=25000 @@ -48,11 +48,11 @@ maxsize=204800 ymalflag = 2 getymal = 0 0 1 * * 1 cveflag = 2 -getcve = 0 3 18 * * * +getcve = 0 5 11 * * * oricveflag = 2 oricvecheck = 0 16 19 * * * getissueflag = 2 -getissue = 0 40 15 * * * +getissue = 0 38 10 * * * issueflag = 2 createissue = 0 7 16 * * * test = 0/10 * * * * * @@ -82,8 +82,8 @@ issuestatistic = 0 19 10 * * * #email = 1499273991@qq.com #redirect_uri = http://119.8.126.102:80/v1/issue/oauth/callback # -------jianjun gitee 配置 -------- -#owner = src-openeuler -owner = cve-test +owner = src-openeuler +#owner = cve-test path = jasper email = 7844966+zhangjianjun_code@user.noreply.gitee.com redirect_uri = http://159.138.2.2:80/v1/issue/oauth/callback @@ -141,13 +141,14 @@ snsuffix = 1002 # Version package excel download address # example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; # openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv -v_pack_20_03_url = "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" +v_pack_20_03_url = "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv;openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv" # Time difference in different time zones sa_timestamp_zone = 28800 [xml] updateinfo_path = download/updateinfo.xml cvrf_upload_path = http://159.138.2.2:9090/ +use_openeuler_num = openEuler-SA-2021-1001 [email] email_name = "${EMAIL_NAME||***}" diff --git a/conf/product_app.conf b/conf/product_app.conf index d02e8263761cc31ca8afbffeccc5e18bb6e68662..0e38f38ab95c60b91a14aa4c162b535bfb961c85 100644 --- a/conf/product_app.conf +++ b/conf/product_app.conf @@ -137,13 +137,14 @@ snsuffix = 1002 # Version package excel download address # example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; # openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv -v_pack_20_03_url = "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv" +v_pack_20_03_url = "openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv;openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv" # Time difference in different time zones sa_timestamp_zone = 28800 [xml] updateinfo_path = download/updateinfo.xml cvrf_upload_path = http://localhost:9090/ +use_openeuler_num = openEuler-SA-2021-1001 [email] email_name = "${EMAIL_NAME||***}" diff --git a/controllers/file.go b/controllers/file.go index 3dfc0b8610062ed2dadfc990f06977235d60e8f3..bad7ec7883176fe3e0ea2d702e0ce084ebfbdaae 100644 --- a/controllers/file.go +++ b/controllers/file.go @@ -84,6 +84,8 @@ func (f *FileController) TriggerCveData() { f.Ctx.WriteString("Error:start time cannot be empty") return } + // proc OpenEulerSaNum + taskhandler.UpdateUseOpenEulerSaNum() //It is time-consuming to generate excel, here is the current limit processing er := models.ExportRecord{} err := er.QueryLast() @@ -268,7 +270,9 @@ func uploadCvrfFile(cvrfFileList map[string][]string, totalFileSlice []string, d } taskhandler.PostFile(indexFilePath, uploadPath) readErr :=taskhandler.ReadWriteFile(updateFilePath, subFileSlice) - logs.Info(readErr) + if readErr != nil { + logs.Error(updateFilePath, readErr) + } taskhandler.PostFile(updateFilePath, uploadPath) totalFileSlice = append(totalFileSlice, indexFilePath) totalFileSlice = append(totalFileSlice, updateFilePath) diff --git a/doc/sql/db_struct.sql b/doc/sql/db_struct.sql index 61d63d1d4cbc3c473d1a51e4cf2d5bfcf8eae1c8..b4f35238e45c908f90e18f1ecaa155610f2fc6b8 100644 --- a/doc/sql/db_struct.sql +++ b/doc/sql/db_struct.sql @@ -50,7 +50,7 @@ CREATE TABLE `cve_cvrf_sa_record` ( UNIQUE KEY `openeuler_sa_num` (`openeuler_sa_num`), UNIQUE KEY `cve_cvrf_md5` (`cur_md5`), KEY `cve_cvrf_sa_record_cve_num` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=15 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=378 DEFAULT CHARSET=utf8; /*Table structure for table `cve_email_list` */ @@ -62,7 +62,7 @@ CREATE TABLE `cve_email_list` ( `email_type` tinyint NOT NULL DEFAULT '1', PRIMARY KEY (`id`), KEY `cve_email_list_email_type` (`email_type`) -) ENGINE=InnoDB AUTO_INCREMENT=9 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=10 DEFAULT CHARSET=utf8; /*Table structure for table `cve_export_record` */ @@ -76,7 +76,7 @@ CREATE TABLE `cve_export_record` ( `state` tinyint NOT NULL DEFAULT '0', PRIMARY KEY (`id`), UNIQUE KEY `file_name` (`file_name`) -) ENGINE=InnoDB AUTO_INCREMENT=162 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=194 DEFAULT CHARSET=utf8; /*Table structure for table `cve_file_hash` */ @@ -87,7 +87,7 @@ CREATE TABLE `cve_file_hash` ( `file_name` varchar(50) DEFAULT NULL, `file_hash` varchar(50) DEFAULT NULL, PRIMARY KEY (`id`) -) ENGINE=InnoDB AUTO_INCREMENT=141 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=418 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_open_euler` */ @@ -121,7 +121,7 @@ CREATE TABLE `cve_git_open_euler` ( KEY `cve_git_open_euler_package_id` (`package_id`), KEY `cve_git_open_euler_version` (`version`), KEY `cve_git_open_euler_table_id` (`table_id`) -) ENGINE=InnoDB AUTO_INCREMENT=5134 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=7493 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_open_euler_table_relate` */ @@ -166,7 +166,7 @@ CREATE TABLE `cve_git_package_info` ( KEY `cve_git_package_info_git_id` (`git_id`), KEY `cve_git_package_info_git_ids` (`git_ids`), KEY `cve_git_package_info_version` (`version`) -) ENGINE=InnoDB AUTO_INCREMENT=4874 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=4890 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_package_table` */ @@ -188,7 +188,7 @@ CREATE TABLE `cve_git_repo_groups` ( `group_name` varchar(255) NOT NULL DEFAULT '', PRIMARY KEY (`group_id`), KEY `cve_git_repo_groups_group_name` (`group_name`) -) ENGINE=InnoDB AUTO_INCREMENT=151 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=153 DEFAULT CHARSET=utf8; /*Table structure for table `cve_git_sub_pack` */ @@ -273,10 +273,11 @@ CREATE TABLE `cve_gite_origin_issue` ( `proc_status` tinyint NOT NULL DEFAULT '0', `inf_product` varchar(255) DEFAULT NULL, `repo_desc` varchar(2048) DEFAULT NULL, + `issue_state` varchar(50) NOT NULL DEFAULT '', PRIMARY KEY (`id`), UNIQUE KEY `issue_id` (`issue_id`), UNIQUE KEY `number` (`number`,`cve_number`,`repo_path`) -) ENGINE=InnoDB AUTO_INCREMENT=9749 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=11177 DEFAULT CHARSET=utf8; /*Table structure for table `cve_gite_repo` */ @@ -297,7 +298,7 @@ CREATE TABLE `cve_gite_repo` ( `delete_time` varchar(255) DEFAULT NULL, PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_group_id` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=108916 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=263949 DEFAULT CHARSET=utf8; /*Table structure for table `cve_gite_repo_branch` */ @@ -328,7 +329,7 @@ CREATE TABLE `cve_gite_repo_member` ( PRIMARY KEY (`repo_id`), KEY `cve_gite_repo_member_group_id` (`group_id`), CONSTRAINT `member_groups_group_id` FOREIGN KEY (`group_id`) REFERENCES `cve_git_repo_groups` (`group_id`) -) ENGINE=InnoDB AUTO_INCREMENT=6766 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=16553 DEFAULT CHARSET=utf8; /*Table structure for table `cve_ip_white` */ @@ -376,7 +377,7 @@ CREATE TABLE `cve_issue_create_record` ( PRIMARY KEY (`id`), KEY `cve_issue_create_record_cve_id` (`cve_id`), KEY `cve_issue_create_record_cve_num` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=1477 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=1517 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_hooks` */ @@ -404,7 +405,7 @@ CREATE TABLE `cve_issue_hooks` ( UNIQUE KEY `cve_issue_hooks_owner_repo_status_un` (`owner`,`repo`,`hook_url`,`status`), KEY `cve_issue_hooks_Cve_id` (`cve_id`), KEY `cve_issue_hooks_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=33 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=37 DEFAULT CHARSET=utf8; /*Table structure for table `cve_issue_repo_whitelist` */ @@ -424,6 +425,21 @@ CREATE TABLE `cve_issue_repo_whitelist` ( KEY `cve_issue_repo_whitelist_version` (`version`) ) ENGINE=InnoDB AUTO_INCREMENT=338 DEFAULT CHARSET=utf8; +/*Table structure for table `cve_issue_statistics_mail_list` */ + +DROP TABLE IF EXISTS `cve_issue_statistics_mail_list`; + +CREATE TABLE `cve_issue_statistics_mail_list` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `email_name` varchar(256) NOT NULL DEFAULT '', + `email_type` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `email_name` (`email_name`) +) ENGINE=InnoDB AUTO_INCREMENT=106 DEFAULT CHARSET=utf8; + /*Table structure for table `cve_issue_template` */ DROP TABLE IF EXISTS `cve_issue_template`; @@ -472,7 +488,7 @@ CREATE TABLE `cve_issue_template` ( UNIQUE KEY `cve_issue_template_num_compone_versio` (`cve_num`,`owned_component`,`owned_version`), KEY `cve_issue_template_cve_id` (`cve_id`), KEY `cve_issue_template_issue_num` (`issue_num`) -) ENGINE=InnoDB AUTO_INCREMENT=2389 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2663 DEFAULT CHARSET=utf8; /*Table structure for table `cve_open_euler_s_a` */ @@ -486,7 +502,7 @@ CREATE TABLE `cve_open_euler_s_a` ( PRIMARY KEY (`openeuler_id`), UNIQUE KEY `openeuler_sa_num` (`openeuler_sa_num`), KEY `cve_open_euler_s_a_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2875 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2925 DEFAULT CHARSET=utf8; /*Table structure for table `cve_open_guss_yaml` */ @@ -501,10 +517,11 @@ CREATE TABLE `cve_open_guss_yaml` ( `create_time` varchar(32) NOT NULL DEFAULT '', `update_time` varchar(32) DEFAULT NULL, `delete_time` varchar(32) DEFAULT NULL, + `cpe_name` varchar(1024) NOT NULL DEFAULT '', PRIMARY KEY (`id`), UNIQUE KEY `package_name_version` (`package_name`,`version`), KEY `cve_open_guss_yaml_version` (`version`) -) ENGINE=InnoDB AUTO_INCREMENT=127 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=144 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_excel` */ @@ -515,7 +532,7 @@ CREATE TABLE `cve_origin_excel` ( `cve_num` varchar(256) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `cve_url` varchar(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `cve_version` varchar(128) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, - `pack_name` varchar(1024) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, + `pack_name` varchar(256) DEFAULT NULL, `score_type` varchar(64) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, `nvd_score` decimal(10,1) DEFAULT NULL, `cve_level` varchar(32) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, @@ -539,8 +556,8 @@ CREATE TABLE `cve_origin_excel` ( `delete_time` datetime DEFAULT NULL, `is_export` tinyint DEFAULT '1', PRIMARY KEY (`cve_id`) USING BTREE, - UNIQUE KEY `cve_origin_excel_num_un` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=380 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; + UNIQUE KEY `cve_origin_excel_num_un` (`cve_num`,`cve_version`,`pack_name`) +) ENGINE=InnoDB AUTO_INCREMENT=443 DEFAULT CHARSET=utf8 ROW_FORMAT=DYNAMIC; /*Table structure for table `cve_origin_upstream` */ @@ -573,7 +590,7 @@ CREATE TABLE `cve_origin_upstream` ( KEY `cve_origin_upstream_cve_packname` (`cve_packname`), KEY `cve_origin_upstream_git_packname` (`git_packname`), KEY `cve_origin_upstream_version` (`version`) -) ENGINE=InnoDB AUTO_INCREMENT=1857 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2148 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config` */ @@ -585,7 +602,7 @@ CREATE TABLE `cve_origin_upstream_config` ( `nodes` varchar(32) DEFAULT NULL, PRIMARY KEY (`conf_id`), KEY `cve_origin_upstream_config_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node` */ @@ -597,7 +614,7 @@ CREATE TABLE `cve_origin_upstream_config_node` ( `operator` varchar(256) DEFAULT NULL, PRIMARY KEY (`node_id`), KEY `cve_origin_upstream_config_node_conf_id` (`conf_id`) -) ENGINE=InnoDB AUTO_INCREMENT=24811 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=26544 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_config_node_cpe` */ @@ -611,7 +628,7 @@ CREATE TABLE `cve_origin_upstream_config_node_cpe` ( `vulner_able` varchar(64) DEFAULT NULL, PRIMARY KEY (`cpe_id`), KEY `cve_origin_upstream_config_node_cpe_node_id` (`node_id`) -) ENGINE=InnoDB AUTO_INCREMENT=767824 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=773120 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_desc` */ @@ -624,7 +641,7 @@ CREATE TABLE `cve_origin_upstream_desc` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`desc_id`), KEY `cve_origin_upstream_desc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_event` */ @@ -639,7 +656,7 @@ CREATE TABLE `cve_origin_upstream_event` ( `description` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`event_id`), KEY `cve_origin_upstream_event_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest` */ @@ -651,7 +668,7 @@ CREATE TABLE `cve_origin_upstream_fix_suggest` ( `detail` varchar(1024) DEFAULT NULL, PRIMARY KEY (`fix_id`), KEY `cve_origin_upstream_fix_suggest_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13135 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14032 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_fix_suggest_ref` */ @@ -689,7 +706,7 @@ CREATE TABLE `cve_origin_upstream_impact` ( `impact` varchar(32) DEFAULT NULL, PRIMARY KEY (`impact_id`), KEY `cve_origin_upstream_impact_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score` */ @@ -705,7 +722,7 @@ CREATE TABLE `cve_origin_upstream_impact_score` ( `score_status` tinyint DEFAULT '1', PRIMARY KEY (`score_id`), KEY `cve_origin_upstream_impact_score_impact_id` (`impact_id`) -) ENGINE=InnoDB AUTO_INCREMENT=26375 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=28169 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v2` */ @@ -734,7 +751,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v2` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v2_id`), KEY `cve_origin_upstream_impact_score_v2_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_impact_score_v3` */ @@ -760,7 +777,7 @@ CREATE TABLE `cve_origin_upstream_impact_score_v3` ( `cve_level` varchar(32) DEFAULT NULL, PRIMARY KEY (`v3_id`), KEY `cve_origin_upstream_impact_score_v3_score_id` (`score_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_poc` */ @@ -777,7 +794,7 @@ CREATE TABLE `cve_origin_upstream_poc` ( `desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`poc_id`), KEY `cve_origin_upstream_poc_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13188 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14085 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_reference` */ @@ -792,7 +809,7 @@ CREATE TABLE `cve_origin_upstream_reference` ( `tags` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`ref_id`), KEY `cve_origin_upstream_reference_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=260775 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=271924 DEFAULT CHARSET=utf8; /*Table structure for table `cve_origin_upstream_vul_type` */ @@ -806,7 +823,7 @@ CREATE TABLE `cve_origin_upstream_vul_type` ( `zh_desc` text CHARACTER SET utf8 COLLATE utf8_general_ci, PRIMARY KEY (`vul_id`), KEY `cve_origin_upstream_vul_type_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=13135 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=14032 DEFAULT CHARSET=utf8; /*Table structure for table `cve_other_user` */ @@ -835,7 +852,7 @@ CREATE TABLE `cve_package` ( `pack_url` varchar(2048) DEFAULT NULL, PRIMARY KEY (`id`), KEY `cve_package_sec_id` (`sec_id`) -) ENGINE=InnoDB AUTO_INCREMENT=53784 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=82349 DEFAULT CHARSET=utf8; /*Table structure for table `cve_package_cpe` */ @@ -850,6 +867,36 @@ CREATE TABLE `cve_package_cpe` ( KEY `cve_package_cpe_packname` (`packname`) ) ENGINE=InnoDB AUTO_INCREMENT=2480 DEFAULT CHARSET=utf8; +/*Table structure for table `cve_sa_file_list` */ + +DROP TABLE IF EXISTS `cve_sa_file_list`; + +CREATE TABLE `cve_sa_file_list` ( + `file_id` bigint NOT NULL AUTO_INCREMENT, + `file_name` varchar(512) NOT NULL DEFAULT '', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`file_id`) +) ENGINE=InnoDB AUTO_INCREMENT=67 DEFAULT CHARSET=utf8; + +/*Table structure for table `cve_sa_number` */ + +DROP TABLE IF EXISTS `cve_sa_number`; + +CREATE TABLE `cve_sa_number` ( + `sa_id` bigint NOT NULL AUTO_INCREMENT, + `openeuler_sa_num` varchar(128) NOT NULL DEFAULT '', + `sa_years` varchar(16) NOT NULL DEFAULT '', + `sa_num` bigint NOT NULL DEFAULT '0', + `status` tinyint NOT NULL DEFAULT '1', + `create_time` varchar(32) NOT NULL DEFAULT '', + `update_time` varchar(32) DEFAULT NULL, + `delete_time` varchar(32) DEFAULT NULL, + PRIMARY KEY (`sa_id`) +) ENGINE=InnoDB AUTO_INCREMENT=37984 DEFAULT CHARSET=utf8; + /*Table structure for table `cve_score` */ DROP TABLE IF EXISTS `cve_score`; @@ -894,7 +941,7 @@ CREATE TABLE `cve_score` ( PRIMARY KEY (`id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_score_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2875 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2925 DEFAULT CHARSET=utf8; /*Table structure for table `cve_score_record` */ @@ -909,7 +956,7 @@ CREATE TABLE `cve_score_record` ( `create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (`id`), KEY `cve_score_record_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=3659 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=3725 DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_notice` */ @@ -935,7 +982,7 @@ CREATE TABLE `cve_security_notice` ( PRIMARY KEY (`sec_id`), UNIQUE KEY `openeuler_id` (`openeuler_id`), KEY `cve_security_notice_cve_id` (`cve_id`) -) ENGINE=InnoDB AUTO_INCREMENT=2875 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2937 DEFAULT CHARSET=utf8; /*Table structure for table `cve_security_reviewer` */ @@ -966,7 +1013,7 @@ CREATE TABLE `cve_spec_error` ( PRIMARY KEY (`id`), UNIQUE KEY `cve_spec_error_cve_num` (`cve_num`), KEY `cve_spec_error_cve_owner` (`cve_owner`) -) ENGINE=InnoDB AUTO_INCREMENT=8226 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=20023 DEFAULT CHARSET=utf8; /*Table structure for table `cve_spec_issue_assigness` */ @@ -1004,10 +1051,11 @@ CREATE TABLE `cve_vuln_center` ( `is_export` tinyint NOT NULL DEFAULT '0', `data_source` tinyint NOT NULL DEFAULT '1', `cve_detail_url` varchar(1024) NOT NULL DEFAULT '', + `organizate_id` tinyint NOT NULL DEFAULT '1', PRIMARY KEY (`cve_id`), UNIQUE KEY `cve_vuln_center_num_pack_v_un` (`cve_num`,`cve_version`,`pack_name`), KEY `cve_vuln_center_cve_num` (`cve_num`) -) ENGINE=InnoDB AUTO_INCREMENT=2881 DEFAULT CHARSET=utf8; +) ENGINE=InnoDB AUTO_INCREMENT=2931 DEFAULT CHARSET=utf8; /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; diff --git a/models/cve.go b/models/cve.go index 46ee1e457e258536cc165204e9908f416fa291ea..46bc17b048629dc0ac6f585a4bdd8ad006969fdc 100644 --- a/models/cve.go +++ b/models/cve.go @@ -175,8 +175,11 @@ func QueryCveCvssV2(scoreId int64) (OriginUpstreamImpactScoreV2, bool) { func QueryCveByNum(cveNum, packName, version string) (VulnCenter, bool) { o := orm.NewOrm() var cve VulnCenter - err := o.Raw("select * from cve_vuln_center where cve_num = ? and pack_name = ? and cve_version = ?", - cveNum, packName, version).QueryRow(&cve) + //err := o.Raw("select * from cve_vuln_center where cve_num = ? and pack_name = ? and cve_version = ?", + // cveNum, packName, version).QueryRow(&cve) + err := o.Raw("select * from cve_vuln_center where cve_num = ? and " + + "pack_name = ? order by cve_id desc limit 1", + cveNum, packName).QueryRow(&cve) if err != nil || cve.CveId == 0 { return cve, false } else { diff --git a/models/modeldb.go b/models/modeldb.go index 520f2bae4f87659a76195f310e48d36176491ee2..5ab2c758701b3eec06c80d3a79def89d19f68ccf 100644 --- a/models/modeldb.go +++ b/models/modeldb.go @@ -39,7 +39,7 @@ type VulnCenter struct { Description string `orm:"size(8192);column(cve_desc)" description:"cve描述"` CveLevel string `orm:"size(32);column(cve_level)" description:"致命(Critical);严重(High);中等(Medium);一般(Low);其他"` Status int8 `orm:"default(0);column(cve_status)" description:"0:cve新增;1:数据已变化；2：已创建issue; 3: 数据创建失败; 4:不符合创建issue条件数据(cve年份不符合要求);5:issue已经创建过，不符合要求，不处理;6:字段为空(评分,描述等)"` - CveVersion string `orm:"size(128);column(cve_version)" description:"cve归属版本"` + CveVersion string `orm:"size(512);column(cve_version)" description:"cve归属版本,版本合并"` RepairTime string `orm:"size(32);column(repair_time)" description:"cve修复时间"` PackName string `orm:"size(512);column(pack_name)" description:"cve对应得包名称"` CveUrl string `orm:"size(2048);column(cve_url)" description:"cve下载链接"` @@ -138,7 +138,7 @@ type IssueTemplate struct { CveId int64 `orm:"index;column(cve_id)"` CveNum string `orm:"size(256);column(cve_num)" description:"cve编号"` OwnedComponent string `orm:"size(256);column(owned_component)" description:"漏洞归属组件"` - OwnedVersion string `orm:"size(256);column(owned_version)" description:"漏洞归属版本"` + OwnedVersion string `orm:"size(512);column(owned_version)" description:"漏洞归属版本"` NVDScore float64 `orm:"digits(10);decimals(1);column(nvd_score)" description:"nvd评分"` OpenEulerScore float64 `orm:"digits(10);decimals(1);column(openeuler_score)" description:"openeuler评分"` NVDVector string `orm:"size(256);column(nvd_vector)" description:"nvd评分向量"` diff --git a/task/issuetask.go b/task/issuetask.go index c9c542f398e0d2644ceaf7cecf0f704942b497b9..2c4a7accce581679b7aa550eece53744d7828ff0 100644 --- a/task/issuetask.go +++ b/task/issuetask.go @@ -130,7 +130,7 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears if seError == nil && se.Id > 0 { models.UpdateIssueStatus(issueValue, 5) logs.Info("The current issue does not need to be processed, it has been processed, cveData: ", issueValue) - ErrorCveStatistics("CVE已经归档无需处理", issueValue, 2) + ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) continue } // Determine whether the issue has been processed @@ -140,7 +140,7 @@ func addUnlimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears goi.State == "已完成" || goi.State == "已拒绝" || goi.IssueState == "已挂起" { models.UpdateIssueStatus(issueValue, 2) logs.Info("The cve data has already been submitted to the issue, no need to submit repeatedly, cveData: ", issueValue) - ErrorCveStatistics("CVE已创建issue, 且已归档", issueValue, 2) + ErrorCveStatistics("CVE已创建过issue, 且已归档", issueValue, 1) continue } } @@ -248,7 +248,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i models.UpdateIssueStatus(issueValue, 5) logs.Info("The current issue does not need to be processed,"+ " it has been processed, cveData: ", issueValue) - ErrorCveStatistics("CVE已经归档无需处理", issueValue, 2) + ErrorCveStatistics("CVE已经归档无需处理", issueValue, 1) continue } // Determine whether the issue has been processed @@ -258,7 +258,7 @@ func addLimitedIssue(beforeTime string, prcnum int, years, toolYears, manYears i goi.State == "已完成" || goi.State == "已拒绝" || goi.IssueState == "已挂起" { models.UpdateIssueStatus(issueValue, 2) logs.Info("cve数据已经已经提交过issue,不需要重复提交, cveData: ", issueValue) - ErrorCveStatistics("CVE已创建issue, 且已归档", issueValue, 2) + ErrorCveStatistics("CVE已创建过issue, 且已归档", issueValue, 1) continue } } diff --git a/taskhandler/cve.go b/taskhandler/cve.go index e8f7e117bf669c7404387d890e57458251cc5c30..af25d611f08e7525c1fd2ae49e4199419fef4a1f 100644 --- a/taskhandler/cve.go +++ b/taskhandler/cve.go @@ -782,10 +782,12 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) if key != "" && len(key) > 1 { pkList := []string{} pkList = append(pkList, key) - pkList = append(pkList, value) + //pkList = append(pkList, value) versionFlag := false gaussFlag := false var goe models.GitPackageInfo + openGausVersion := make([]string, 0) + openEulerVersion := make([]string, 0) if value != "" && len(value) > 0 { versionList := strings.Split(value, ",") if len(versionList) > 0 { @@ -794,12 +796,13 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) opy := models.OpenGussYaml{PackageName: key, Version: ver} openErr := models.GetOpengaussYaml(&opy, "PackageName", "Version") if openErr == nil && opy.Id > 0 { + openGausVersion = append(openGausVersion, ver) gaussFlag = true } goe, ok = models.QueryCveOpeneulerDetaildataByName(key, ver) if ok { versionFlag = true - break + openEulerVersion = append(openEulerVersion, ver) } else { logs.Error("未查询到对应的仓库数据, data: ", cveData) } @@ -827,6 +830,9 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) } CveRes, err := models.QueryCveByNum(cveData.CveNum, key, value) if err { + retVersion := AddCveVersion(organizationID, openGausVersion, + openEulerVersion, CveRes.CveVersion) + pkList = append(pkList, retVersion) lockx.Lock() ok, err := UpdateCveGroups(cveData, cveRef, openeulernum, CveRes, cveDesc, cveScV3, goe, scopeType, cveScV2, pkList, organizationID) @@ -837,6 +843,9 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) return false, errors.New("数据错误，暂时不处理") } } else { + retVersion := AddCveVersion(organizationID, openGausVersion, + openEulerVersion, "") + pkList = append(pkList, retVersion) lockx.Lock() ok, err := InsertCveGroups(cveData, cveRef, openeulernum, cveDesc, cveScV3, goe, scopeType, cveScV2, pkList, organizationID) @@ -853,6 +862,33 @@ func GenCveVuler(cveData models.OriginUpstream, cveRef string, openeulernum int) return true, nil } +func AddCveVersion(organizationID int8, openGausVersion []string, + openEulerVersion []string, cveVersion string) (retVersion string) { + retVersion = cveVersion + if organizationID == 2 { + for _, versx := range openGausVersion { + if retVersion != "" && len(retVersion) > 0 { + if !strings.Contains(retVersion, versx) { + retVersion = retVersion + "," + versx + } + } else { + retVersion = versx + } + } + } else { + for _, versx := range openEulerVersion { + if retVersion != "" && len(retVersion) > 0 { + if !strings.Contains(retVersion, versx) { + retVersion = retVersion + "," + versx + } + } else { + retVersion = versx + } + } + } + return retVersion +} + func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYears int) (bool, error) { if cveData.CveNum == "" || len(cveData.CveNum) == 0 { logs.Error("当前数据cveNum 为空暂不处理,data: ", cveData) @@ -895,6 +931,9 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe } CveRes, err := models.QueryCveByNum(cveData.CveNum, cveData.PackName, cveData.CveVersion) if err { + retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, + []string{cveData.CveVersion}, CveRes.CveVersion) + cveData.CveVersion = retVersion lockx.Lock() ok, err := UpdateExcelCveGroups(cveData, cveRef, openeulerNum, CveRes, goe, organizationID) lockx.Unlock() @@ -904,6 +943,9 @@ func SyncCveVuler(cveData models.OriginExcel, cveRef string, openeulerNum, manYe return false, errors.New("数据错误，暂时不处理") } } else { + retVersion := AddCveVersion(organizationID, []string{cveData.CveVersion}, + []string{cveData.CveVersion}, "") + cveData.CveVersion = retVersion lockx.Lock() ok, err := InsertCveExcelGroups(cveData, cveRef, openeulerNum, goe, organizationID) lockx.Unlock() @@ -1010,7 +1052,9 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c specCharList := []string{"

", "\n", "\r", "\t"} vul.Description = RemoveSubstring(lop.BriefIntroduction, specCharList) vul.Status = cveStatus - vul.CveVersion = RemoveSubstring(lop.Version, specCharList) + retVersion := AddCveVersion(1, []string{}, + strings.Split(RemoveSubstring(lop.Version, specCharList), ","), "") + vul.CveVersion = retVersion vul.RepairTime = "" vul.PackName = cveData.RepoPath vul.CveUrl = cveRef + cveData.CveNumber @@ -1228,9 +1272,12 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c vul.Description = RemoveSubstring(lop.BriefIntroduction, specCharList) } vul.Status = cveStatus - if vul.CveVersion == "" || len(vul.CveVersion) < 1 { - vul.CveVersion = RemoveSubstring(lop.Version, specCharList) - } + retVersion := AddCveVersion(1, []string{}, + strings.Split(RemoveSubstring(lop.Version, specCharList), ","), vul.CveVersion) + vul.CveVersion = retVersion + //if vul.CveVersion == "" || len(vul.CveVersion) < 1 { + // vul.CveVersion = RemoveSubstring(lop.Version, specCharList) + //} vul.RepairTime = "" if vul.PackName == "" || len(vul.PackName) < 1 { vul.PackName = cveData.RepoPath diff --git a/taskhandler/cvrf.go b/taskhandler/cvrf.go index 245632687097280615033443ce1e51229b1b5a63..83b8d02310c1782e0507b2bef28b69cf65984fa9 100644 --- a/taskhandler/cvrf.go +++ b/taskhandler/cvrf.go @@ -9,10 +9,7 @@ import ( "errors" "fmt" "github.com/astaxie/beego/logs" - "io" "io/ioutil" - "mime/multipart" - "net/http" "os" "sort" "strconv" @@ -544,13 +541,15 @@ func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, noteDescription.Type = "General" noteDescription.Ordinal = "3" noteDescription.XmlLang = "en" - noteDescription.Note = v.Description + descriptionStr := strings.ReplaceAll(v.Description, "\n\n", "\r\n\r\n") + noteDescription.Note = descriptionStr note = append(note, noteDescription) var noteTopic Note noteTopic.Title = "Topic" noteTopic.Type = "General" noteTopic.Ordinal = "4" noteTopic.XmlLang = "en" + topic := strings.ReplaceAll(topic, "\n\n", "\r\n\r\n") noteTopic.Note = topic note = append(note, noteTopic) var noteSeverity Note @@ -583,13 +582,14 @@ func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, notex := make([]Note, 0) for _, te := range note { if te.Title == "Description" { - dSplit := strings.Split(v.Description, "Security Fix(es):") + descriptionStr := strings.ReplaceAll(v.Description, "\n\n", "\r\n\r\n") + dSplit := strings.Split(descriptionStr, "Security Fix(es):") if len(dSplit) > 1 { if !strings.Contains(te.Note, dSplit[0]) { te.Note = dSplit[0] + te.Note } if !strings.Contains(te.Note, dSplit[1]) { - te.Note += dSplit[1] + "\r\n" + te.Note += dSplit[1] } } te.Note = te.Note @@ -598,6 +598,7 @@ func BuilddocumentNotes(cvrfsa *CvrfSa, v models.ExcelExport, vcn := strings.Join(componentInfo.CveNum, ";\n") theme, err := models.GetCanExportTheme(vcn, v.InfluenceComponent, affectBranch) if err == nil && len(theme) > 1 { + theme = strings.ReplaceAll(theme, "\n\n", "\r\n\r\n") if branchCount <= 1 { te.Note = theme } else if branchCount == 2 { @@ -1254,41 +1255,41 @@ func RecordCrvfInfo(fileName, filex string, fixFlag int8) error { return nil } +// File Upload func PostFile(filename string, targetUrl string) error { - bodyBuf := &bytes.Buffer{} - bodyWriter := multipart.NewWriter(bodyBuf) - fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filename) - if err != nil { - logs.Error("error writing to buffer, ", err) - return err - } - fh, err := os.Open(filename) - if err != nil { - logs.Error("error opening file, ", err, ", filename: ", filename) - return err - } - _, err = io.Copy(fileWriter, fh) - if err != nil { - logs.Error(err) - return err - } - contentType := bodyWriter.FormDataContentType() - bodyWriter.Close() - resp, err := http.Post(targetUrl, contentType, bodyBuf) - if err != nil { - return err - } - defer resp.Body.Close() - resp_body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return err - } - logs.Info(resp.Status) - logs.Info(string(resp_body)) + //bodyBuf := &bytes.Buffer{} + //bodyWriter := multipart.NewWriter(bodyBuf) + //fileWriter, err := bodyWriter.CreateFormFile("uploadfile", filename) + //if err != nil { + // logs.Error("error writing to buffer, ", err) + // return err + //} + //fh, err := os.Open(filename) + //if err != nil { + // logs.Error("error opening file, ", err, ", filename: ", filename) + // return err + //} + //_, err = io.Copy(fileWriter, fh) + //if err != nil { + // logs.Error(err) + // return err + //} + //contentType := bodyWriter.FormDataContentType() + //bodyWriter.Close() + //resp, err := http.Post(targetUrl, contentType, bodyBuf) + //if err != nil { + // return err + //} + //defer resp.Body.Close() + //resp_body, err := ioutil.ReadAll(resp.Body) + //if err != nil { + // return err + //} + //logs.Info(resp.Status) + //logs.Info(string(resp_body)) return nil } - func ReadWriteFile(filePath string, fileSlice []string) error { if len(fileSlice) == 0 { return errors.New("file content does not exist") @@ -1315,4 +1316,4 @@ func ReadWriteFile(filePath string, fileSlice []string) error { logs.Error(err) } return nil -} \ No newline at end of file +} diff --git a/taskhandler/excel.go b/taskhandler/excel.go index 15b7bfaad9dc83e09d4ac861c91c4dfe3a64e813..31ef5f8778c207610bcb943a3bf993435104c256 100644 --- a/taskhandler/excel.go +++ b/taskhandler/excel.go @@ -673,6 +673,24 @@ func addXmlData(canExport []models.ExcelExport, cvexml *[]CveXml, affectBranch s } } +func UpdateUseOpenEulerSaNum() { + useOpenEulerNum := beego.AppConfig.String("xml::use_openeuler_num") + if len(useOpenEulerNum) > 1 { + useOpenEulerNumSlice := strings.Split(useOpenEulerNum, ",") + for _, use := range useOpenEulerNumSlice { + var sax models.SaNumber + sax.OpenEulerSANum = use + sax.Status = 1 + models.GetSaNumber(&sax, "OpenEulerSANum", "Status") + if sax.SaId > 0 { + sax.Status = 2 + sax.UpdateTime = common.GetCurTime() + models.UpdateSaNumber(&sax, "status") + } + } + } +} + func GetOpenEulerSaNum() string { var sa models.SaNumber curYears := strconv.Itoa(time.Now().Year()) @@ -837,9 +855,9 @@ func BranchCvrfData(v models.ExcelExport, cvrfsa = CvrfSa{} } else { var componentInfo ComponentInfo - componentInfo = componentMap[v.OwnedComponent] + componentInfo = componentMap[v.InfluenceComponent] componentInfo.UpdateFlag = 1 - componentMap[v.OwnedComponent] = componentInfo + componentMap[v.InfluenceComponent] = componentInfo } } BuildCvrfXml(&cvrfsa, v, affectBranch, cvrfFileList, componentMap, pkg)