diff --git a/cve-vulner-manager/conf/app.conf b/cve-vulner-manager/conf/app.conf index 6442fad50aef8377f48a41e6cd471a19ecd6ab0c..c77e78d4045ecc3e37872aef3996581feef0a64d 100644 --- a/cve-vulner-manager/conf/app.conf +++ b/cve-vulner-manager/conf/app.conf @@ -54,11 +54,11 @@ ParseOpenlookengYamlTask = "30 4 * * 0-6" [mysql] -dbhost = "${DB_URI||127.0.0.1}" -dbport = 3310 -dbuser = "${DB_USER||cve}" -dbpwd = "${DB_PWD||***}" -dbname = oneline_cvevulner +dbhost = "${DB_URI||10.0.0.161}" +dbport = 3306 +dbuser = "${DB_USER||root}" +dbpwd = "${DB_PWD||123456}" +dbname = cvevulner_online dbprefix = cve_ maxidle = 30 maxconn = 3000 @@ -139,7 +139,7 @@ releaseUnaffectedCve = 0 0 11 * * 1 #redirect_uri = http://119.8.126.102:80/v1/issue/oauth/callback # -------jianjun gitee 配置 -------- #owner = src-openeuler -owner = cve-test +owner = src-openeuler path = jasper email = 7844966+zhangjianjun_code@user.noreply.gitee.com redirect_uri = http://159.138.2.2:80/v1/issue/oauth/callback @@ -153,7 +153,7 @@ client_secret = "${GITEE_CLIENT_SECRET||****}" password = "${GITEE_PASSWORD||****}" # git token -git_token = "${GITEE_TOKEN||xxx}" +git_token = "${GITEE_TOKEN||82bff85208414136c0ef726f6e76d0dc}" [hook] hookpwd = "${HOOK_PWD||***}" @@ -182,7 +182,7 @@ cve_number_t = 1990 # Create an issue's repo whitelist;1: open; 2: close issue_whitelist = 2 # List of affected branches： openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-SP3 -affected_branchs = "openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP2,openEuler-20.03-LTS-SP3,openEuler-22.03-LTS" +affected_branchs = "openEuler-20.03-LTS-SP1,openEuler-20.03-LTS-SP3,openEuler-22.03-LTS,openEuler-22.03-LTS-SP1,openEuler-22.03-LTS-SP2" abandoned_branchs = "openEuler-20.03-LTS,openEuler-21.03,openEuler-21.09,openEuler-20.09" # Close the highest privilege of issue close_issue_privilege = 2 @@ -227,7 +227,8 @@ snsuffix = 1002 # example: openEuler-20.03-LTS@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS.csv; # openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv # public sa format: openEuler-20.03-LTS@https;openEuler-20.03-LTS-SP1@https -v_pack_20_03_url = "openEuler-20.03-LTS-SP1@https;openEuler-20.03-LTS-SP3@https;openEuler-22.03-LTS@https" +v_pack_20_03_url = "openEuler-20.03-LTS-SP1@https;openEuler-20.03-LTS-SP3@https;openEuler-22.03-LTS@https;openEuler-22.03-LTS-SP1@https;openEuler-22.03-LTS-SP2@https" +release_date_of_version = "openEuler-22.03-LTS-SP2:2023-06-30;" # Time difference in different time zones sa_timestamp_zone = 28800 unaffect_year = 2018 @@ -264,8 +265,8 @@ git_mindspore_token = "${GITEE_MINDSPORE_TOKEN||xxx}" mindspore_version = "master,r1.10,r1.7,r1.8,r1.9" [obs] -access_key_id = "${OBS_KEY_ID||xxx}" -secret_access_key = "${OBS_SECRET_KEY||xxx}" +access_key_id = "${OBS_KEY_ID||IKWJN7MYDQWL6IRFOD9C}" +secret_access_key = "${OBS_SECRET_KEY||TMmWXHhoPVU6jtB2RzHVLdDMQ9hi7uuiQNeyG7Qg}" endpoint = obs.ap-southeast-1.myhuaweicloud.com bucket = openeuler-cve-cvrf upload_cvrf_dir = "cve-manager-cvrf/" diff --git a/cve-vulner-manager/conf/product_app.conf b/cve-vulner-manager/conf/product_app.conf index a66e8bf3f7241d3beaeda9411698fd1de7900ebd..8a40d7fc20d895d9c0eafec9c34c31a4f42d040e 100644 --- a/cve-vulner-manager/conf/product_app.conf +++ b/cve-vulner-manager/conf/product_app.conf @@ -221,6 +221,7 @@ snsuffix = 1002 # openEuler-20.03-LTS-SP1@http://119.3.219.20:88/mkb/obs_update_info/openEuler-20.03-LTS-SP1.csv # public sa format: openEuler-20.03-LTS@https;openEuler-20.03-LTS-SP1@https v_pack_20_03_url = "openEuler-20.03-LTS-SP1@https;openEuler-20.03-LTS-SP3@https;openEuler-22.03-LTS@https;openEuler-22.03-LTS-SP1@https;openEuler-22.03-LTS-SP2@https" +release_date_of_version = "openEuler-22.03-LTS-SP2:2023-06-30;" # Time difference in different time zones sa_timestamp_zone = 28810 unaffect_year = 2018 diff --git a/cve-vulner-manager/controllers/cve.go b/cve-vulner-manager/controllers/cve.go index 01f5b248b89779773e47ab1af709ed7626215e66..8fb9c4b8e71ab6d594809accf7cdaaa0ccd6672d 100644 --- a/cve-vulner-manager/controllers/cve.go +++ b/cve-vulner-manager/controllers/cve.go @@ -191,7 +191,7 @@ func (u *CveAllIssueController) Get() { CveNum: issues.CveNum, IssueNum: issues.IssueNum, Version: issues.OwnedVersion, - Component: issues.OwnedComponent, + Component: issues.Repo, OpeneulerScore: issues.OpeneulerScore, NvdScore: issues.NvdScore, Branch: issues.AffectedVersion, diff --git a/cve-vulner-manager/main.go b/cve-vulner-manager/main.go index 64c5458aceb91cbe58f466bf8e785307de5e6dcb..0410593232f8fe253b62ecc7bbfe68ccda9d345f 100644 --- a/cve-vulner-manager/main.go +++ b/cve-vulner-manager/main.go @@ -10,6 +10,7 @@ import ( _ "cvevulner/models" _ "cvevulner/routers" "cvevulner/task" + "cvevulner/taskhandler" ) func init() { @@ -32,6 +33,8 @@ func main() { return } + taskhandler.InitReleaseDate() + // Initialize a scheduled task taskOk := task.InitTask() if !taskOk { diff --git a/cve-vulner-manager/task/issuetask.go b/cve-vulner-manager/task/issuetask.go index 74e2e71d10b772f971ddb0e66a329359f8a51760..14bf83205f46a054ad6af2f242023bce58405ffb 100644 --- a/cve-vulner-manager/task/issuetask.go +++ b/cve-vulner-manager/task/issuetask.go @@ -559,13 +559,7 @@ func ProcUpdateIssue(issueValue models.VulnCenter, accessToken, owner string) er it.CveBrief = issueValue.Description it.CveLevel = issueValue.CveLevel if it.Assignee == "" && issueValue.OrganizationID == 1 { - if assignee, assErr := taskhandler.GetCollaboratorInfo(accessToken, owner, path); assErr != nil || assignee == "" { - logs.Error("ProcUpdateIssue, Get the repo: owner:", owner, - ", path:", path, ", Analyst failed, ", "assErr:", assErr, - ", CveNum: ", issueValue.CveNum, ", Create an issue without a maintainer") - } else { - it.Assignee = assignee - } + it.Assignee = taskhandler.GetAssignerOfOpeneuler(path) } _, err := taskhandler.UpdateIssueToGit(accessToken, owner, path, issueValue, it) if err != nil && err.Error() != "Recreate issue" { @@ -640,11 +634,7 @@ func EulerIssue(issueValue models.VulnCenter, accessToken, owner, path, assignee return errors.New("CVE描述缺失") } - assignee, err = taskhandler.GetCollaboratorInfo(accessToken, owner, path) - if err != nil { - logs.Error("ProcIssue, obtain collaborator error: ", err) - return err - } + assignee = taskhandler.GetAssignerOfOpeneuler(path) branchList := make([]string, 0) errBrands := errors.New("") diff --git a/cve-vulner-manager/taskhandler/assist.go b/cve-vulner-manager/taskhandler/assist.go index 15fb3a1f7b8e357960428d6c75f32ba7ed7fa85e..50b8cbacac7474c6237c229c4467c761ea39cffc 100644 --- a/cve-vulner-manager/taskhandler/assist.go +++ b/cve-vulner-manager/taskhandler/assist.go @@ -3,17 +3,25 @@ package taskhandler import ( "encoding/json" "errors" + "fmt" + "net/http" "os" "sort" "strings" + "sync" + "time" "github.com/astaxie/beego" "github.com/astaxie/beego/logs" + "github.com/opensourceways/server-common-lib/utils" "cvevulner/common" "cvevulner/util" ) +var mutex sync.Mutex +var assignerOfOpeneulerRepoCache map[string]string + func GetOautToken(gt GiteeToken) { url := "https://gitee.com/oauth/token" var req util.RequestInfo @@ -73,6 +81,132 @@ func GetCollaboratorInfo(accessToken string, owner string, path string) (string, return "", errors.New("参数错误") } +func GetAssignerOfOpeneuler(repo string) string { + mutex.Lock() + defer mutex.Unlock() + keyOfDate := "key_of_date" + today := time.Now().Format("20060102") + + initCache := func() { + assignerOfOpeneulerRepoCache = make(map[string]string) + assignerOfOpeneulerRepoCache[keyOfDate] = today + initAssignerCache() + } + + if len(assignerOfOpeneulerRepoCache) == 0 { + initCache() + } + + date, ok := assignerOfOpeneulerRepoCache[keyOfDate] + if !ok || date != today { + initCache() + } + + if assigner, ok := assignerOfOpeneulerRepoCache[repo]; ok { + return assigner + } + + return "" +} + +func initAssignerCache() { + cli := utils.NewHttpClient(3) + for _, sig := range getSig() { + url := fmt.Sprintf("https://www.openeuler.org/api-dsapi/query/sig/repo/committers?community=openeuler&sig=%s", sig) + + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error("new request of sig %s err: %s", sig, err.Error()) + continue + } + r, _, err := cli.Download(request) + if err != nil { + logs.Error("get assigner of sig %s err: %s", sig, err.Error()) + continue + } + + var res ResCommitter + if err = json.Unmarshal(r, &res); err != nil { + logs.Error("unmarshal of sig %s err: %s", sig, err.Error()) + continue + } + + var sigAssigner string + if len(res.Data.Maintainers) > 0 { + sigAssigner = res.Data.Maintainers[0] + } + + for _, v := range res.Data.CommitterDetails { + if !strings.Contains(v.Repo, "src-openeuler") { + continue + } + + var assigner string + if len(v.GiteeId) > 0 { + assigner = v.GiteeId[0] + } else { + assigner = sigAssigner + } + + split := strings.Split(v.Repo, "/") + if len(split) < 2 { + continue + } + + assignerOfOpeneulerRepoCache[split[1]] = assigner + } + } +} + +type ResContent struct { + Type string `json:"type"` + Name string `json:"name"` +} + +type ResCommitter struct { + Data struct { + Maintainers []string `json:"maintainers"` + CommitterDetails []struct { + GiteeId []string `json:"gitee_id"` + Repo string `json:"repo"` + } `json:"committerDetails"` + } `json:"data"` +} + +func getSig() []string { + url := "https://gitee.com/api/v5/repos/openeuler/community/contents/sig" + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + logs.Error("new request of sig url error: %s ", err.Error()) + + return nil + } + + cli := utils.NewHttpClient(3) + var res []ResContent + r, _, err := cli.Download(request) + if err != nil { + logs.Error("get sig of openeuler error: %s", err.Error()) + + return nil + } + + if err = json.Unmarshal(r, &res); err != nil { + logs.Error("unmarshal sig error: %s", err.Error()) + + return nil + } + + var sig []string + for _, v := range res { + if v.Type == "dir" { + sig = append(sig, v.Name) + } + } + + return sig +} + func GetBranchesInfo(accessToken, owner, path string, organizationID int8) ([]string, error) { BrandVerSlice := make([]string, 0) switch organizationID { diff --git a/cve-vulner-manager/taskhandler/createissue.go b/cve-vulner-manager/taskhandler/createissue.go index bb806c11216513516bc4d6f9b6d0dd981ca294be..646f153b19c6a67b8cff84935bc30c0f1f293140 100644 --- a/cve-vulner-manager/taskhandler/createissue.go +++ b/cve-vulner-manager/taskhandler/createissue.go @@ -168,14 +168,14 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, it.CveNum = cve.CveNum _, assignLoginList, _ := GetRepoMember(accessToken, owner, path) templateErr := models.GetIssueTemplateByColName(&it, "CveId", "CveNum") - if cve.OrganizationID == 1 { - sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} - specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") - if specError == nil && sia.Id > 0 { - it.Assignee = sia.Assignee - assignee = sia.Assignee - } - } + //if cve.OrganizationID == 1 { + // sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} + // specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") + // if specError == nil && sia.Id > 0 { + // it.Assignee = sia.Assignee + // assignee = sia.Assignee + // } + //} assigneeGite := "" if cve.DataSource == 4 && cve.CveId == it.CveId && len(it.IssueNum) > 2 { logs.Info("Current data is synchronized") @@ -238,9 +238,9 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, if len(assigneeGite) > 1 { it.Assignee = assigneeGite } else { - if cve.OrganizationID == 1 && len(assignLoginList) > 0 { - it.Assignee = assignLoginList[0] - } + //if cve.OrganizationID == 1 && len(assignLoginList) > 0 { + // it.Assignee = assignLoginList[0] + //} } requestBody := CreateIssueBody(accessToken, owner, path, it.Assignee, cve, sc, OpenEulerScore, score, labels, it, 1, it.IssueType, "", brandArray) @@ -319,9 +319,9 @@ func CreateIssueToGit(accessToken, owner, path, assignee string, return "", errors.New("Recreate issue") } } - if len(assignLoginList) > 0 { - assignee = assignLoginList[0] - } + //if len(assignLoginList) > 0 { + // assignee = assignLoginList[0] + //} } if it.TemplateId > 0 { models.DeleteIssueTemplate(it.TemplateId) @@ -524,13 +524,13 @@ func UpdateIssueToGit(accessToken, owner, path string, } } //labels := its.IssueLabel - if cve.OrganizationID == 1 { - sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} - specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") - if specError == nil && sia.Id > 0 { - its.Assignee = sia.Assignee - } - } + //if cve.OrganizationID == 1 { + // sia := models.SpecIssueAssigness{PackageName: cve.PackName, Status: 1} + // specError := models.GetSpecIssueAssignee(&sia, "package_name", "status") + // if specError == nil && sia.Id > 0 { + // its.Assignee = sia.Assignee + // } + //} labels := "" if its.IssueLabel != "" && len(its.IssueLabel) > 1 { labels = its.IssueLabel diff --git a/cve-vulner-manager/taskhandler/cve.go b/cve-vulner-manager/taskhandler/cve.go index d219b226758a51ee65f3a359372d06b950b0c69d..d15e17ec65449aaa881ebb631379be1630e25249 100644 --- a/cve-vulner-manager/taskhandler/cve.go +++ b/cve-vulner-manager/taskhandler/cve.go @@ -1484,7 +1484,7 @@ func InsertIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c vul.RepairTime = "" } vul.PackName = cveData.RepoPath - vul.RepoName = lop.Components + vul.RepoName = cveData.RepoPath vul.CveUrl = cveRef + cveData.CveNumber vul.IsExport = 0 vul.DataSource = 4 @@ -1705,7 +1705,7 @@ func UpdateIssueCveGroups(cveData models.GiteOriginIssue, lop models.Loophole, c if vul.CveUrl == "" || len(vul.CveUrl) < 1 { vul.CveUrl = cveRef + cveData.CveNumber } - vul.RepoName = lop.Components + vul.RepoName = cveData.RepoPath vul.OrganizationID = cveData.OrganizationID accessToken := "" owner, accessToken = common.GetOwnerAndToken(vul.CveNum, vul.OrganizationID) diff --git a/cve-vulner-manager/taskhandler/excel.go b/cve-vulner-manager/taskhandler/excel.go index f4263a055fec8c74de9359e4c59526aaa9051d51..d130c0d83b47f6a45a54dd79de35ce3ea33318a3 100644 --- a/cve-vulner-manager/taskhandler/excel.go +++ b/cve-vulner-manager/taskhandler/excel.go @@ -32,6 +32,8 @@ const UNAFFECTCVRFKEY = "unaffectcvrfkey" const FIXEDFLAGE = 1 const UNAFFECTFLAG = 2 +var releaseDate map[string]int64 + //CveExcel Excel export client type CveExcel struct { ExcelName string //excel name @@ -1366,6 +1368,14 @@ func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, } else { pt = mt } + + // Branches that have been fixed before the release no longer publish announcements + if releaseTimeOfVersion, ok := releaseDate[affectBranch]; ok { + if pt < releaseTimeOfVersion { + continue + } + } + if pt >= startTime && pt <= releaseTime { if v["base"].(map[string]interface{})["label"].(string) == affectBranch { //if v["base"].(map[string]interface{})["repo"].(map[string]interface{})["path"] == repo { @@ -1394,3 +1404,12 @@ func getRepoIssueAllPR(affectBranch, token, owner, repo string, startTime, } return } + +func InitReleaseDate() { + releaseDate = make(map[string]int64) + releaseDateConfig := beego.AppConfig.DefaultString("excel::release_date_of_version", "") + for _, v := range strings.Split(strings.Trim(releaseDateConfig, ";"), ";") { + split := strings.Split(v, ":") + releaseDate[split[0]] = util.TimeStrToInt(split[1], "2006-01-02") + 3600*24 + } +}