From 8c5de221a7125752dfd62be3adbfe31b979cbd9f Mon Sep 17 00:00:00 2001 From: vegbir Date: Sat, 7 Sep 2024 19:10:22 +0800 Subject: [PATCH] optimize containerInfo 1. reconstruct containerInfo 2. abstract systemd & cgroupfs Signed-off-by: vegbir --- pkg/core/typedef/cgroup/cgroupfs/driver.go | 15 + pkg/core/typedef/cgroup/driver.go | 10 +- pkg/core/typedef/cgroup/systemd/driver.go | 47 ++- pkg/core/typedef/containerinfo.go | 188 ++++++----- pkg/core/typedef/engine.go | 69 ++++ pkg/core/typedef/nrirawpod.go | 372 ++------------------- pkg/core/typedef/podinfo.go | 64 ++-- pkg/core/typedef/rawpod.go | 28 +- pkg/podmanager/podmanager.go | 52 ++- 9 files changed, 379 insertions(+), 466 deletions(-) create mode 100644 pkg/core/typedef/engine.go diff --git a/pkg/core/typedef/cgroup/cgroupfs/driver.go b/pkg/core/typedef/cgroup/cgroupfs/driver.go index 4256deb..87fb748 100644 --- a/pkg/core/typedef/cgroup/cgroupfs/driver.go +++ b/pkg/core/typedef/cgroup/cgroupfs/driver.go @@ -35,3 +35,18 @@ func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string { return filepath.Join(constant.KubepodsCgroup, qosClass, constant.PodCgroupNamePrefix+id) } + +func (d *Driver) GetNRIContainerCgroupPath(nriCgroupPath string) string { + // When using cgroupfs as cgroup driver and isula, docker, containerd as container runtime: + // 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3/88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec + // 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3/88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec + // 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3/88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec + return nriCgroupPath +} + +func (d *Driver) ConcatContainerCgroup(podCgroupPath, prefix, containerID string) string { + if prefix != "" { + prefix = prefix + "-" + } + return filepath.Join(podCgroupPath, prefix+containerID) +} diff --git a/pkg/core/typedef/cgroup/driver.go b/pkg/core/typedef/cgroup/driver.go index b8cd4d5..4d5225b 100644 --- a/pkg/core/typedef/cgroup/driver.go +++ b/pkg/core/typedef/cgroup/driver.go @@ -21,6 +21,8 @@ import ( type Driver interface { Name() string ConcatPodCgroupPath(qosClass string, id string) string + ConcatContainerCgroup(podCgroupPath, prefix, containerID string) string + GetNRIContainerCgroupPath(nriCgroupPath string) string } var driver Driver = &cgroupfs.Driver{} @@ -43,6 +45,10 @@ func ConcatPodCgroupPath(qosClass, id string) string { return driver.ConcatPodCgroupPath(qosClass, id) } -func ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string { - return driver.ConcatPodCgroupPath(podCgroupPath, containerScope) +func GetNRIContainerCgroupPath(nriCgroupPath string) string { + return driver.GetNRIContainerCgroupPath(nriCgroupPath) +} + +func ConcatContainerCgroup(podCgroupPath, prefix, containerID string) string { + return driver.ConcatContainerCgroup(podCgroupPath, prefix, containerID) } diff --git a/pkg/core/typedef/cgroup/systemd/driver.go b/pkg/core/typedef/cgroup/systemd/driver.go index 740b7ba..de8f956 100644 --- a/pkg/core/typedef/cgroup/systemd/driver.go +++ b/pkg/core/typedef/cgroup/systemd/driver.go @@ -20,7 +20,10 @@ import ( "isula.org/rubik/pkg/common/constant" ) -const Name = "systemd" +const ( + Name = "systemd" + cgroupFileExt = ".scope" +) type Driver struct{} @@ -46,6 +49,44 @@ func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string { strings.Join([]string{prefix, constant.PodCgroupNamePrefix + strings.Replace(id, "-", "_", -1) + suffix}, "-")) } -func (d *Driver) ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string { - return filepath.Join(podCgroupPath, containerScope+".scope") +func (d *Driver) GetNRIContainerCgroupPath(nriCgroupPath string) string { + // When using systemd as cgroup driver: + // 1. The Burstable path looks like: + // kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec.scope + // 2. The BestEffort path is in the form: + // kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec.scope + // 3. The Guaranteed path is in the form: + // kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec.scope + + // what we get from nri container: kubepods-besteffort-pod7631cab3_4785_4a70_a4f3_03505fb28b64.slice:cri-containerd:d4d54e90e1c55e71910e5196d4e65be39ff8c5fb39a2c3e662893ff2ab9b42cd + + // 1. we parse cgroupPath and get + // parent: kubepods-besteffort-pod7631cab3_4785_4a70_a4f3_03505fb28b64.slice + // prefix: cri-containerd + // containerID: d4d54e90e1c55e71910e5196d4e65be39ff8c5fb39a2c3e662893ff2ab9b42cd + parts := strings.Split(nriCgroupPath, ":") + var parent, prefix, id = parts[0], parts[1], parts[2] + + // 2. the last segment of the path must be cri-containerd-d4d54e90e1c55e71910e5196d4e65be39ff8c5fb39a2c3e662893ff2ab9b42cd.scope + last := prefix + "-" + id + cgroupFileExt + + // 3. parse parent to obtain the upper directory + parentParts := strings.Split(parent, "-") + var upperDir string + // for the Guaranteed, we get 2 parts: kubepods, pod9d8d5026_5f11_4530_b929_c11f833027c2.slice + // for the others, we get 3 parts: kubepods, besteffort, pod7631cab3_4785_4a70_a4f3_03505fb28b64.slice + if len(parentParts) > 2 { + // This means upper directory should contain a kubepods-besteffort.slice or kubepods-burstable.slice + upperDir = parentParts[0] + "-" + parentParts[1] + ".slice" + } + + const topDir = constant.KubepodsCgroup + ".slice" + return filepath.Join(topDir, upperDir, parent, last) +} + +func (d *Driver) ConcatContainerCgroup(podCgroupPath, prefix, containerID string) string { + if prefix != "" { + prefix = prefix + "-" + } + return filepath.Join(podCgroupPath, prefix+containerID+cgroupFileExt) } diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index ec04ed8..3c61f08 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -16,112 +16,136 @@ package typedef import ( "fmt" - "path/filepath" - "strings" - "sync" "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/core/typedef/cgroup" ) -// ContainerEngineType indicates the type of container engine -type ContainerEngineType int8 - -const ( - // UNDEFINED means undefined container engine - UNDEFINED ContainerEngineType = iota - // DOCKER means docker container engine - DOCKER - // CONTAINERD means containerd container engine - CONTAINERD - // ISULAD means isulad container engine - ISULAD - // CRIO means crio container engine - CRIO -) +// ContainerInfo contains the interested information of container +type ContainerInfo struct { + cgroup.Hierarchy + Name string `json:"name"` + ID string `json:"id"` + RequestResources ResourceMap `json:"requests,omitempty"` + LimitResources ResourceMap `json:"limits,omitempty"` + PodSandboxId string `json:"podisandid,omitempty"` // id of the sandbox which can uniquely determine a pod +} + +// DeepCopy returns deepcopy object. +func (cont *ContainerInfo) DeepCopy() *ContainerInfo { + copyObject := *cont + copyObject.LimitResources = cont.LimitResources.DeepCopy() + copyObject.RequestResources = cont.RequestResources.DeepCopy() + return ©Object +} + +type ContainerConfig struct { + rawCont *RawContainer + nriCont *NRIRawContainer + request ResourceMap + limit ResourceMap + podCgroupPath string +} + +type ConfigOpt func(b *ContainerConfig) -var ( - supportEnginesPrefixMap = map[ContainerEngineType]string{ - DOCKER: "docker://", - CONTAINERD: "containerd://", - ISULAD: "iSulad://", - CRIO: "cri-o://", - } - currentContainerEngines = UNDEFINED - setContainerEnginesOnce sync.Once - containerEngineScopes = map[ContainerEngineType]string{ - DOCKER: "docker", - CONTAINERD: "cri-containerd", - ISULAD: "isulad", - CRIO: "crio", +func WithRawContainer(cont *RawContainer) ConfigOpt { + return func(conf *ContainerConfig) { + conf.rawCont = cont } -) +} -// Support returns true when the container uses the container engine -func (engine *ContainerEngineType) Support(cont *RawContainer) bool { - if *engine == UNDEFINED { - return false +func WithNRIContainer(cont *NRIRawContainer) ConfigOpt { + return func(conf *ContainerConfig) { + conf.nriCont = cont } - return strings.HasPrefix(cont.status.ContainerID, engine.Prefix()) } -// Prefix returns the ID prefix of the container engine -func (engine *ContainerEngineType) Prefix() string { - prefix, ok := supportEnginesPrefixMap[*engine] - if !ok { - return "" +func WithRequest(req ResourceMap) ConfigOpt { + return func(conf *ContainerConfig) { + conf.request = req } - return prefix } -// ContainerInfo contains the interested information of container -type ContainerInfo struct { - cgroup.Hierarchy - Name string `json:"name"` - ID string `json:"id"` - RequestResources ResourceMap `json:"requests,omitempty"` - LimitResources ResourceMap `json:"limits,omitempty"` - // PodSandboxId means the id of the pod which the container belongs to - PodSandboxId string `json:"podisandid,omitempty"` +func WithLimit(limit ResourceMap) ConfigOpt { + return func(conf *ContainerConfig) { + conf.limit = limit + } } -func containerPath(id, podCgroupPath string) string { - if cgroup.Type() == constant.CgroupDriverSystemd { - return filepath.Join(podCgroupPath, containerEngineScopes[currentContainerEngines]+"-"+id+".scope") +func WithPodCgroup(path string) ConfigOpt { + return func(conf *ContainerConfig) { + conf.podCgroupPath = path } - // In the case of cgroupfs, the path of crio contains a special prefix - if containerEngineScopes[currentContainerEngines] == constant.ContainerEngineCrio { - return filepath.Join(podCgroupPath, constant.ContainerEngineCrio+"-"+id) +} + +func NewContainerInfo(opts ...ConfigOpt) *ContainerInfo { + var ( + conf = &ContainerConfig{} + ci = &ContainerInfo{} + ) + for _, opt := range opts { + opt(conf) } - return filepath.Join(podCgroupPath, id) + + if err := fromRawContainer(ci, conf.rawCont); err != nil { + fmt.Printf("failed to parse raw container: %v", err) + } + fromNRIContainer(ci, conf.nriCont) + fromPodCgroupPath(ci, conf.podCgroupPath) + + if conf.request != nil { + ci.RequestResources = conf.request + } + + if conf.limit != nil { + ci.LimitResources = conf.limit + } + + return ci } -// NewContainerInfo creates a ContainerInfo instance -func NewContainerInfo(id, podCgroupPath string, rawContainer *RawContainer) *ContainerInfo { - requests, limits := rawContainer.GetResourceMaps() - return &ContainerInfo{ - Name: rawContainer.status.Name, - ID: id, - Hierarchy: cgroup.Hierarchy{Path: containerPath(id, podCgroupPath)}, - RequestResources: requests, - LimitResources: limits, +func fromRawContainer(ci *ContainerInfo, rawCont *RawContainer) error { + if rawCont == nil { + return nil } + requests, limits := rawCont.GetResourceMaps() + id, err := rawCont.GetRealContainerID() + if err != nil { + return fmt.Errorf("failed to parse container ID: %v", err) + } + if id == "" { + return fmt.Errorf("empty container id") + } + + ci.Name = rawCont.status.Name + ci.ID = id + ci.RequestResources = requests + ci.LimitResources = limits + return nil } -func getEngineFromContainerID(containerID string) { - for engine, prefix := range supportEnginesPrefixMap { - if strings.HasPrefix(containerID, prefix) { - currentContainerEngines = engine - fmt.Printf("The container engine is %v\n", strings.Split(currentContainerEngines.Prefix(), ":")[0]) - return - } +// convert NRIRawContainer structure to ContainerInfo structure +func fromNRIContainer(ci *ContainerInfo, nriCont *NRIRawContainer) { + if nriCont == nil { + return } + ci.ID = nriCont.Id + ci.Hierarchy = cgroup.Hierarchy{ + Path: cgroup.GetNRIContainerCgroupPath(nriCont.Linux.GetCgroupsPath()), + } + ci.Name = nriCont.Name + ci.PodSandboxId = nriCont.PodSandboxId } -// DeepCopy returns deepcopy object. -func (cont *ContainerInfo) DeepCopy() *ContainerInfo { - copyObject := *cont - copyObject.LimitResources = cont.LimitResources.DeepCopy() - copyObject.RequestResources = cont.RequestResources.DeepCopy() - return ©Object +func fromPodCgroupPath(ci *ContainerInfo, podCgroupPath string) { + if podCgroupPath == "" { + return + } + // TODO : don't need to judge cgroup driver + var prefix = containerEngineScopes[currentContainerEngines] + if cgroup.Type() == constant.CgroupDriverCgroupfs && currentContainerEngines != CRIO { + prefix = "" + } + ci.Hierarchy = cgroup.Hierarchy{Path: cgroup.ConcatContainerCgroup(podCgroupPath, prefix, ci.ID)} } diff --git a/pkg/core/typedef/engine.go b/pkg/core/typedef/engine.go new file mode 100644 index 0000000..2906c26 --- /dev/null +++ b/pkg/core/typedef/engine.go @@ -0,0 +1,69 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2021-2024. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2024-09-07 +// Description: This file is used for container engines + +package typedef + +import ( + "strings" + "sync" +) + +// ContainerEngineType indicates the type of container engine +type ContainerEngineType int8 + +const ( + // UNDEFINED means undefined container engine + UNDEFINED ContainerEngineType = iota + // DOCKER means docker container engine + DOCKER + // CONTAINERD means containerd container engine + CONTAINERD + // ISULAD means isulad container engine + ISULAD + // CRIO means crio container engine + CRIO +) + +var ( + supportEnginesPrefixMap = map[ContainerEngineType]string{ + DOCKER: "docker://", + CONTAINERD: "containerd://", + ISULAD: "iSulad://", + CRIO: "cri-o://", + } + containerEngineScopes = map[ContainerEngineType]string{ + DOCKER: "docker", + CONTAINERD: "cri-containerd", + ISULAD: "isulad", + CRIO: "crio", + } + currentContainerEngines = UNDEFINED + setContainerEnginesOnce sync.Once +) + +// Support returns true when the container uses the container engine +func (engine *ContainerEngineType) Support(cont *RawContainer) bool { + if *engine == UNDEFINED { + return false + } + return strings.HasPrefix(cont.status.ContainerID, engine.Prefix()) +} + +// Prefix returns the ID prefix of the container engine +func (engine *ContainerEngineType) Prefix() string { + prefix, ok := supportEnginesPrefixMap[*engine] + if !ok { + return "" + } + return prefix +} diff --git a/pkg/core/typedef/nrirawpod.go b/pkg/core/typedef/nrirawpod.go index d060923..991c2cd 100644 --- a/pkg/core/typedef/nrirawpod.go +++ b/pkg/core/typedef/nrirawpod.go @@ -15,19 +15,13 @@ package typedef import ( - "bufio" "encoding/json" "fmt" - "io" - "os" - "path/filepath" - "strings" "github.com/containerd/nri/pkg/api" - "isula.org/rubik/pkg/common/constant" + v1 "k8s.io/api/core/v1" + "isula.org/rubik/pkg/core/typedef/cgroup" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" ) type ( @@ -37,65 +31,26 @@ type ( NRIRawPod api.PodSandbox ) -const ( - // nri Container Annotations "kubectl.kubernetes.io/last-applied-configuration" means container applied config - containerAppliedConfiguration string = "kubectl.kubernetes.io/last-applied-configuration" - // /proc/self/cgroup file - procSelfCgroupFile = "/proc/self/cgroup" - - containerSpec string = "containers" - resourcesSpec string = "resources" - nanoToMicro float64 = 1000 - fileMode os.FileMode = 0666 -) - // convert NRIRawPod structure to PodInfo structure func (pod *NRIRawPod) ConvertNRIRawPod2PodInfo() *PodInfo { if pod == nil { return nil } + requests, limits := pod.GetResourceMaps() return &PodInfo{ Hierarchy: cgroup.Hierarchy{ - Path: pod.CgroupPath(), + Path: pod.Linux.CgroupParent, }, - Name: pod.Name, - UID: pod.Uid, - Namespace: pod.Namespace, - IDContainersMap: make(map[string]*ContainerInfo, 0), - Annotations: pod.Annotations, - Labels: pod.Labels, - ID: pod.Id, - } -} - -// get pod Qos -func (pod *NRIRawPod) GetQosClass() string { - var podQosClass string - podQosClass = "Guaranteed" - if strings.Contains(pod.Linux.CgroupsPath, "burstable") { - podQosClass = "Burstable" - } - if strings.Contains(pod.Linux.CgroupsPath, "besteffort") { - podQosClass = "BestEffort" + Name: pod.Name, + UID: pod.Uid, + Namespace: pod.Namespace, + IDContainersMap: make(map[string]*ContainerInfo, 0), + Annotations: pod.Annotations, + Labels: pod.Labels, + ID: pod.Id, + nriContainerRequest: requests, + nriContainerLimit: limits, } - return podQosClass -} - -// get pod cgroupPath -func (pod *NRIRawPod) CgroupPath() string { - id := pod.Uid - - qosClassPath := "" - switch corev1.PodQOSClass(pod.GetQosClass()) { - case corev1.PodQOSGuaranteed: - case corev1.PodQOSBurstable: - qosClassPath = strings.ToLower(string(corev1.PodQOSBurstable)) - case corev1.PodQOSBestEffort: - qosClassPath = strings.ToLower(string(corev1.PodQOSBestEffort)) - default: - return "" - } - return cgroup.ConcatPodCgroupPath(qosClassPath, id) } // get pod running state @@ -111,292 +66,33 @@ func (pod *NRIRawPod) ID() string { return string(pod.Uid) } -// convert NRIRawContainer structure to ContainerInfo structure -func (container *NRIRawContainer) ConvertNRIRawContainer2ContainerInfo() *ContainerInfo { - if container == nil { - return nil - } - requests, limits := container.GetResourceMaps() - return &ContainerInfo{ - Hierarchy: cgroup.Hierarchy{ - Path: container.CgroupPath(), - }, - Name: container.Name, - ID: container.Id, - RequestResources: requests, - LimitResources: limits, - PodSandboxId: container.PodSandboxId, - } -} - -// get container cgroupPath -func (container *NRIRawContainer) CgroupPath() string { - var path string - /* - When using systemd cgroup driver with burstable qos: - kubepods-burstable-podbb29f378_0c50_4da4_b070_be919e350db2.slice:crio:a575c8505d48fd0f75488fcf979dea7a693633e99709bb82308af54f3bafb186 - convert to: - "kubepods.slice/kubepods-burstable.slice/kubepods-burstable-pod84d0ae01_83a0_42a7_8990_abbb16e59923.slice/crio-66ff3a44a254533880e6b50e8fb52e0311c9158eb66426ae244066a4f11b26e5.scope" - - When using cgroupfs as cgroup driver and isula, docker, containerd as runtime wich burstable qos: - /kubepods/burstable/poda168d109-4d40-4c50-8f89-957b9b0dc5d6/75082fa9e4783ecf3fc2e1ada7cd08fd2dd20d001d36e579e28e3cb00d312ad4 - convert to: - kubepods/burstable/pod42736679-4475-43cf-afb4-e3744f4352fd/88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec - - When using cgroupfs as cgroup drvier and crio as container runtime wich burstable qos: - /kubepods/burstable/poda168d109-4d40-4c50-8f89-957b9b0dc5d6/crio-75082fa9e4783ecf3fc2e1ada7cd08fd2dd20d001d36e579e28e3cb00d312ad4 - convert to: - kubepods/burstable/pod42736679-4475-43cf-afb4-e3744f4352fd/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec - */ - - /* - When using cgroupfs as cgroup driver and isula, docker, containerd as container runtime: - 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3/88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec - 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3/88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec - 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3/88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec - - When using cgroupfs as cgroup driver and crio as container runtime: - 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec - 2. The BestEffort path is in the form: kubepods/besteffort/pod34152897-dbaf-11ea-8cb9-0653660051c3/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec - 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec - - When using systemd as cgroup driver: - 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec.scope - 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec.scope - 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/crio-88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec.scope - */ - qosClassPath := "" - switch corev1.PodQOSClass(container.getQos()) { - case corev1.PodQOSGuaranteed: - case corev1.PodQOSBurstable: - qosClassPath = strings.ToLower(string(corev1.PodQOSBurstable)) - case corev1.PodQOSBestEffort: - qosClassPath = strings.ToLower(string(corev1.PodQOSBestEffort)) - default: - return "" - } - - if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join( - constant.KubepodsCgroup+".slice", - container.getPodCgroupDir(), - containerEngineScopes[currentContainerEngines]+"-"+container.Id+".scope", - ) - default: - path = "" - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - - path = filepath.Join( - constant.KubepodsCgroup+".slice", - constant.KubepodsCgroup+"-"+qosClassPath+".slice", - container.getPodCgroupDir(), - containerEngineScopes[currentContainerEngines]+"-"+container.Id+".scope", - ) - default: - path = "" - } - - } - - } else { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join( - constant.KubepodsCgroup, - container.getPodCgroupDir(), - container.Id, - ) - - case constant.ContainerEngineCrio: - path = filepath.Join( - constant.KubepodsCgroup, - container.getPodCgroupDir(), - containerEngineScopes[currentContainerEngines]+"-"+container.Id, - ) - default: - path = "" - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join( - constant.KubepodsCgroup, - qosClassPath, - container.getPodCgroupDir(), - container.Id, - ) - case constant.ContainerEngineCrio: - path = filepath.Join( - constant.KubepodsCgroup, - qosClassPath, - container.getPodCgroupDir(), - containerEngineScopes[currentContainerEngines]+"-"+container.Id, - ) - - default: - path = "" - } - } - } - return path -} - -// get container's pod's cgroup dir -func (container *NRIRawContainer) getPodCgroupDir() string { - var podPath string - if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd { - podPath = strings.Split(container.Linux.CgroupsPath, ":")[0] - } else { - pathInfo := strings.Split(container.Linux.CgroupsPath, "/") - podPath = pathInfo[len(pathInfo)-2] - } - return podPath -} - -// get Qos through NRIRawContainer -func (container *NRIRawContainer) getQos() string { - var podQosClass string - podQosClass = "Guaranteed" - if strings.Contains(container.Linux.CgroupsPath, "burstable") { - podQosClass = "Burstable" - } - if strings.Contains(container.Linux.CgroupsPath, "besteffort") { - podQosClass = "BestEffort" +func (pod *NRIRawPod) GetResourceMaps() (map[string]ResourceMap, map[string]ResourceMap) { + const containerAppliedConfiguration = "kubectl.kubernetes.io/last-applied-configuration" + configurations := pod.Annotations[containerAppliedConfiguration] + if configurations == "" { + fmt.Printf("empty resource map in pod %v\n", pod.Uid) + return nil, nil } - return podQosClass -} - -// AppliedConfiguration define container applied configure -type AppliedConfiguration struct { - ApiVersion string - Kind string - Metadata interface{} - Spec map[string][]map[string]interface{} -} - -// Resources define container resource info -type Resources struct { - Limits Limits - Requests Requests -} - -// Limits define container resource limit info -type Limits struct { - Memory string - Cpu float64 -} - -// Requests define container resource request info -type Requests struct { - Memory string - Cpu float64 -} - -// ResourceInfo define get resource interface -type ResourceInfo interface { - getCpuInfo() float64 - getMemoryInfo() float64 -} - -// get container cpu request info -func (r *Requests) getCpuInfo() float64 { - return r.Cpu -} - -// get container memory request info -func (r *Requests) getMemoryInfo() float64 { - var converter = func(value *resource.Quantity) float64 { - return float64(value.MilliValue()) / 1000 - } - - q, _ := resource.ParseQuantity(r.Memory) - - return converter(&q) -} - -// get container cpu limit info -func (r *Limits) getCpuInfo() float64 { - return r.Cpu -} - -// get container memory limit info -func (r *Limits) getMemoryInfo() float64 { - var converter = func(value *resource.Quantity) float64 { - return float64(value.MilliValue()) / 1000 + rawPod := &RawPod{} + err := json.Unmarshal([]byte(configurations), rawPod) + if err != nil { + fmt.Printf("failed to unmarshal resource map %v: %v\n", configurations, err) + return nil, nil } + var requests, limits = map[string]ResourceMap{}, map[string]ResourceMap{} - q, _ := resource.ParseQuantity(r.Memory) - - return converter(&q) -} - -// get container request and limit info from container applied configuration -func (container *NRIRawContainer) GetResourceMaps() (ResourceMap, ResourceMap) { - configurations := container.Annotations[containerAppliedConfiguration] - containerConf := &AppliedConfiguration{} - _ = json.Unmarshal([]byte(configurations), containerConf) - resourceInfo := &Resources{} - if r, ok := containerConf.Spec[containerSpec]; ok { - for _, containerSpec := range r { - if containerSpec["name"] == container.Name { - if containerResourceSpec, ok := containerSpec[resourcesSpec]; ok { - if resource, err := json.Marshal(containerResourceSpec); err != nil { - fmt.Printf("get container %v Resource failed ", container.Id) - } else { - if err = json.Unmarshal(resource, resourceInfo); err != nil { - fmt.Printf("container %v data format error", container.Id) - } else { - fmt.Printf("get container %v Resource success", container.Id) - } - } - } else { - fmt.Printf("container %v spec resources info not exist", container.Id) - } - } else { - continue - } + resourceMapConvert := func(rl v1.ResourceList) ResourceMap { + const milli = 1000 + return ResourceMap{ + ResourceCPU: float64(rl.Cpu().MilliValue()) / milli, + ResourceMem: float64(rl.Memory().MilliValue()) / milli, // memory size in bytes } - } else { - fmt.Printf("container %v spec containers info not exist", container.Id) } - iterator := func(res ResourceInfo) ResourceMap { - results := make(ResourceMap) - results[ResourceCPU] = res.getCpuInfo() - results[ResourceMem] = res.getMemoryInfo() - return results - } - return iterator(&resourceInfo.Requests), iterator(&resourceInfo.Limits) -} - -// get current container engine -func getEngineFromCgroup() { - file, err := os.OpenFile(procSelfCgroupFile, os.O_RDONLY, fileMode) - if err != nil { - return - } - defer file.Close() - reader := bufio.NewReader(file) - for { - line, _, err := reader.ReadLine() - s := strings.Split(string(line), "/") - containerEngine := strings.Split(s[len(s)-1], "-")[0] - for engine, prefix := range containerEngineScopes { - if containerEngine == prefix { - currentContainerEngines = engine - break - } - } - if err == io.EOF { - break - } + for _, cont := range rawPod.Spec.Containers { + requests[cont.Name] = resourceMapConvert(cont.Resources.Requests) + limits[cont.Name] = resourceMapConvert(cont.Resources.Limits) } + return requests, limits } diff --git a/pkg/core/typedef/podinfo.go b/pkg/core/typedef/podinfo.go index 656bc3c..17bb66e 100644 --- a/pkg/core/typedef/podinfo.go +++ b/pkg/core/typedef/podinfo.go @@ -22,14 +22,15 @@ import ( // PodInfo represents pod type PodInfo struct { cgroup.Hierarchy - Name string `json:"name"` - UID string `json:"uid"` - Namespace string `json:"namespace"` - IDContainersMap map[string]*ContainerInfo `json:"containers,omitempty"` - Annotations map[string]string `json:"annotations,omitempty"` - Labels map[string]string `json:"labels,omitempty"` - // ID means id of the pod in sandbox but not uid - ID string `json:"id,omitempty"` + Name string `json:"name"` + UID string `json:"uid"` + Namespace string `json:"namespace"` + IDContainersMap map[string]*ContainerInfo `json:"containers,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + ID string `json:"id,omitempty"` // id of the sandbox container in pod + nriContainerRequest map[string]ResourceMap + nriContainerLimit map[string]ResourceMap } // NewPodInfo creates the PodInfo instance @@ -50,42 +51,49 @@ func (pod *PodInfo) DeepCopy() *PodInfo { if pod == nil { return nil } - var ( - contMap map[string]*ContainerInfo - annoMap map[string]string - labelMap map[string]string - ) + + var copy = *pod // nil is different from empty value in golang if pod.IDContainersMap != nil { - contMap = make(map[string]*ContainerInfo) + contMap := make(map[string]*ContainerInfo) for id, cont := range pod.IDContainersMap { contMap[id] = cont.DeepCopy() } + copy.IDContainersMap = contMap } if pod.Annotations != nil { - annoMap = make(map[string]string) + annoMap := make(map[string]string) for k, v := range pod.Annotations { annoMap[k] = v } + copy.Annotations = annoMap } if pod.Labels != nil { - labelMap = make(map[string]string) + labelMap := make(map[string]string) for k, v := range pod.Labels { labelMap[k] = v } + copy.Labels = labelMap } - return &PodInfo{ - Name: pod.Name, - UID: pod.UID, - Hierarchy: pod.Hierarchy, - Namespace: pod.Namespace, - Annotations: annoMap, - Labels: labelMap, - IDContainersMap: contMap, + if pod.nriContainerLimit != nil { + limits := make(map[string]ResourceMap) + for k, v := range pod.nriContainerLimit { + limits[k] = v.DeepCopy() + } + copy.nriContainerLimit = limits + } + + if pod.nriContainerRequest != nil { + requests := make(map[string]ResourceMap) + for k, v := range pod.nriContainerRequest { + requests[k] = v.DeepCopy() + } + copy.nriContainerRequest = requests } + return © } // Offline is used to determine whether the pod is offline @@ -109,3 +117,11 @@ func (pod *PodInfo) Offline() bool { func (pod *PodInfo) Online() bool { return !pod.Offline() } + +func (pod *PodInfo) GetNriContainerRequest() map[string]ResourceMap { + return pod.nriContainerRequest +} + +func (pod *PodInfo) GetNriContainerLimit() map[string]ResourceMap { + return pod.nriContainerLimit +} diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index b653c71..ed11c6b 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -96,7 +96,7 @@ var k8sQosClass = map[corev1.PodQOSClass]string{ // CgroupPath returns cgroup path of raw pod // handle different combinations of cgroupdriver and pod qos and container runtime -func (pod *RawPod) CgroupPath() (res string) { +func (pod *RawPod) CgroupPath() string { id := string(pod.UID) if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" { id = configHash @@ -142,17 +142,16 @@ func (pod *RawPod) ExtractContainerInfos() map[string]*ContainerInfo { } // 2. generate ID-Container mapping - podCgroupPath := pod.CgroupPath() for _, rawContainer := range nameRawContainersMap { - id, err := rawContainer.GetRealContainerID() - if err != nil { - fmt.Printf("failed to parse container ID: %v\n", err) + ci := NewContainerInfo( + WithRawContainer(rawContainer), + WithPodCgroup(pod.CgroupPath()), + ) + if ci.ID == "" { + fmt.Printf("failed to parse id from raw container\n") continue } - if id == "" { - continue - } - idContainersMap[id] = NewContainerInfo(id, podCgroupPath, rawContainer) + idContainersMap[ci.ID] = ci } return idContainersMap } @@ -167,7 +166,6 @@ func (cont *RawContainer) GetRealContainerID() (string, error) { `fixContainerEngine` is only executed when `getRealContainerID` is called for the first time */ setContainerEnginesOnce.Do(func() { - getEngineFromCgroup() _, exist := supportEnginesPrefixMap[currentContainerEngines] if !exist { getEngineFromContainerID(cont.status.ContainerID) @@ -215,3 +213,13 @@ func (m ResourceMap) DeepCopy() ResourceMap { } return res } + +func getEngineFromContainerID(containerID string) { + for engine, prefix := range supportEnginesPrefixMap { + if strings.HasPrefix(containerID, prefix) { + currentContainerEngines = engine + fmt.Printf("The container engine is %v\n", strings.Split(currentContainerEngines.Prefix(), ":")[0]) + return + } + } +} diff --git a/pkg/podmanager/podmanager.go b/pkg/podmanager/podmanager.go index d415018..4135328 100644 --- a/pkg/podmanager/podmanager.go +++ b/pkg/podmanager/podmanager.go @@ -285,23 +285,48 @@ func (manager *PodManager) addNRIPodFunc(pod *typedef.NRIRawPod) { manager.tryAddNRIPod(podInfo) } +func parseNRIContainer(manager *PodManager, container *typedef.NRIRawContainer) *typedef.ContainerInfo { + pod := manager.getPodBySandboxId(container.PodSandboxId) + if pod == nil { + fmt.Printf("failed to find pod by sandbox id %v\n", container.PodSandboxId) + return nil + } + + opts := []typedef.ConfigOpt{ + typedef.WithNRIContainer(container), + } + if req, existed := pod.GetNriContainerRequest()[container.Name]; existed { + opts = append(opts, typedef.WithRequest(req)) + } + if limit, existed := pod.GetNriContainerLimit()[container.Name]; existed { + opts = append(opts, typedef.WithLimit(limit)) + } + return typedef.NewContainerInfo(opts...) +} + // addNRIContainerFunc handles add nri container event func (manager *PodManager) addNRIContainerFunc(container *typedef.NRIRawContainer) { - containerInfo := container.ConvertNRIRawContainer2ContainerInfo() + ci := parseNRIContainer(manager, container) + if ci == nil { + return + } + // TODO: add logics to update pod's container + manager.Pods.Lock() for _, pod := range manager.Pods.Pods { - if containerInfo.PodSandboxId == pod.ID { - pod.IDContainersMap[containerInfo.ID] = containerInfo + if container.PodSandboxId == pod.ID { + pod.IDContainersMap[ci.ID] = ci manager.Publish(typedef.INFOADD, pod.DeepCopy()) + break } } + manager.Pods.Unlock() } // sync to podCache after remove container func (manager *PodManager) removeNRIContainerFunc(container *typedef.NRIRawContainer) { - containerInfo := container.ConvertNRIRawContainer2ContainerInfo() for _, pod := range manager.Pods.Pods { - if containerInfo.PodSandboxId == pod.ID { - delete(pod.IDContainersMap, containerInfo.ID) + if container.PodSandboxId == pod.ID { + delete(pod.IDContainersMap, container.Id) } } } @@ -424,7 +449,11 @@ func (manager *PodManager) nripodssync(pods []*typedef.NRIRawPod) { func (manager *PodManager) nricontainerssync(containers []*typedef.NRIRawContainer) { var newContainers []*typedef.ContainerInfo for _, container := range containers { - newContainers = append(newContainers, container.ConvertNRIRawContainer2ContainerInfo()) + ci := parseNRIContainer(manager, container) + if ci == nil { + continue + } + newContainers = append(newContainers, ci) } manager.Pods.syncContainers2Pods(newContainers) } @@ -472,3 +501,12 @@ func (manager *PodManager) ListPodsWithOptions(options ...api.ListOption) map[st } return pods } + +func (manager *PodManager) getPodBySandboxId(sandboxId string) *typedef.PodInfo { + for _, pod := range manager.ListPodsWithOptions() { + if sandboxId == pod.ID { + return pod + } + } + return nil +} -- Gitee