From fe585042518f7c1a1da09028fe56177fa573ebbc Mon Sep 17 00:00:00 2001 From: vegbir Date: Wed, 4 Sep 2024 10:35:50 +0800 Subject: [PATCH] rubik: set container engine when getting container id reason: bugfix: `/proc/self/cgroup` is not a sure way to get container engine because of the different cgroup path format. So we have two approaches to get container engines: `proc/self/cgroup` or containerdID passed by k8s. reconstruct: We abstract the cgroup driver, i.e., systemd &cgroupfs, to simplify the code for splicing cgroup paths. Currently, only the path to the apiserver has been simplified, and the nri method needs to be reconstructed. Signed-off-by: vegbir --- pkg/core/typedef/cgroup/cgroupfs/driver.go | 37 +++++++++ pkg/core/typedef/cgroup/common.go | 5 -- pkg/core/typedef/cgroup/driver.go | 48 +++++++++++ pkg/core/typedef/cgroup/systemd/driver.go | 51 ++++++++++++ pkg/core/typedef/containerinfo.go | 34 ++++---- pkg/core/typedef/nrirawpod.go | 63 +-------------- pkg/core/typedef/rawpod.go | 92 ++++++---------------- 7 files changed, 175 insertions(+), 155 deletions(-) create mode 100644 pkg/core/typedef/cgroup/cgroupfs/driver.go create mode 100644 pkg/core/typedef/cgroup/driver.go create mode 100644 pkg/core/typedef/cgroup/systemd/driver.go diff --git a/pkg/core/typedef/cgroup/cgroupfs/driver.go b/pkg/core/typedef/cgroup/cgroupfs/driver.go new file mode 100644 index 0000000..4256deb --- /dev/null +++ b/pkg/core/typedef/cgroup/cgroupfs/driver.go @@ -0,0 +1,37 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2024-09-03 +// Description: This file is used for cgroupfs driver + +package cgroupfs + +import ( + "path/filepath" + + "isula.org/rubik/pkg/common/constant" +) + +const Name = "cgroupfs" + +type Driver struct{} + +func (d *Driver) Name() string { + return Name +} + +func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string { + // When using cgroupfs as cgroup driver: + // 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3 + // 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3 + // 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3 + + return filepath.Join(constant.KubepodsCgroup, qosClass, constant.PodCgroupNamePrefix+id) +} diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go index 668f951..5252e7d 100644 --- a/pkg/core/typedef/cgroup/common.go +++ b/pkg/core/typedef/cgroup/common.go @@ -27,11 +27,6 @@ import ( var rootDir = constant.DefaultCgroupRoot var cgroupDriver = constant.CgroupDriverCgroupfs -// SetCgroupDriver is the setter of global cgroup driver -func SetCgroupDriver(driver string) { - cgroupDriver = driver -} - // GetCgroupDriver is the getter of global cgroup driver func GetCgroupDriver() string { return cgroupDriver diff --git a/pkg/core/typedef/cgroup/driver.go b/pkg/core/typedef/cgroup/driver.go new file mode 100644 index 0000000..b8cd4d5 --- /dev/null +++ b/pkg/core/typedef/cgroup/driver.go @@ -0,0 +1,48 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2024-09-03 +// Description: This file is used for cgroup driver + +package cgroup + +import ( + "isula.org/rubik/pkg/core/typedef/cgroup/cgroupfs" + "isula.org/rubik/pkg/core/typedef/cgroup/systemd" +) + +type Driver interface { + Name() string + ConcatPodCgroupPath(qosClass string, id string) string +} + +var driver Driver = &cgroupfs.Driver{} + +// SetCgroupDriver is the setter of global cgroup driver +func SetCgroupDriver(driverTyp string) { + cgroupDriver = driverTyp + switch driverTyp { + case systemd.Name: + driver = &systemd.Driver{} + case cgroupfs.Name: + driver = &cgroupfs.Driver{} + } +} + +func Type() string { + return driver.Name() +} +func ConcatPodCgroupPath(qosClass, id string) string { + return driver.ConcatPodCgroupPath(qosClass, id) +} + +func ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string { + return driver.ConcatPodCgroupPath(podCgroupPath, containerScope) +} diff --git a/pkg/core/typedef/cgroup/systemd/driver.go b/pkg/core/typedef/cgroup/systemd/driver.go new file mode 100644 index 0000000..740b7ba --- /dev/null +++ b/pkg/core/typedef/cgroup/systemd/driver.go @@ -0,0 +1,51 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2024-09-03 +// Description: This file is used for system cgroup driver + +package systemd + +import ( + "path/filepath" + "strings" + + "isula.org/rubik/pkg/common/constant" +) + +const Name = "systemd" + +type Driver struct{} + +func (d *Driver) Name() string { + return Name +} + +func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string { + // When using systemd as cgroup driver: + // 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice + // 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice + // 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/ + const suffix = ".slice" + var ( + prefix = constant.KubepodsCgroup + podPath = constant.KubepodsCgroup + suffix + ) + if qosClass != "" { + podPath = filepath.Join(podPath, constant.KubepodsCgroup+"-"+qosClass+suffix) + prefix = strings.Join([]string{prefix, qosClass}, "-") + } + return filepath.Join(podPath, + strings.Join([]string{prefix, constant.PodCgroupNamePrefix + strings.Replace(id, "-", "_", -1) + suffix}, "-")) +} + +func (d *Driver) ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string { + return filepath.Join(podCgroupPath, containerScope+".scope") +} diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index 841c800..ec04ed8 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -85,33 +85,30 @@ type ContainerInfo struct { PodSandboxId string `json:"podisandid,omitempty"` } +func containerPath(id, podCgroupPath string) string { + if cgroup.Type() == constant.CgroupDriverSystemd { + return filepath.Join(podCgroupPath, containerEngineScopes[currentContainerEngines]+"-"+id+".scope") + } + // In the case of cgroupfs, the path of crio contains a special prefix + if containerEngineScopes[currentContainerEngines] == constant.ContainerEngineCrio { + return filepath.Join(podCgroupPath, constant.ContainerEngineCrio+"-"+id) + } + return filepath.Join(podCgroupPath, id) +} + // NewContainerInfo creates a ContainerInfo instance func NewContainerInfo(id, podCgroupPath string, rawContainer *RawContainer) *ContainerInfo { - scopeName := containerEngineScopes[currentContainerEngines] requests, limits := rawContainer.GetResourceMaps() - var path string - if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join(podCgroupPath, scopeName+"-"+id+".scope") - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join(podCgroupPath, id) - case constant.ContainerEngineCrio: - path = filepath.Join(podCgroupPath, scopeName+"-"+id) - } - } return &ContainerInfo{ Name: rawContainer.status.Name, ID: id, - Hierarchy: cgroup.Hierarchy{Path: path}, + Hierarchy: cgroup.Hierarchy{Path: containerPath(id, podCgroupPath)}, RequestResources: requests, - LimitResources: limits} + LimitResources: limits, + } } -func fixContainerEngine(containerID string) { +func getEngineFromContainerID(containerID string) { for engine, prefix := range supportEnginesPrefixMap { if strings.HasPrefix(containerID, prefix) { currentContainerEngines = engine @@ -119,7 +116,6 @@ func fixContainerEngine(containerID string) { return } } - currentContainerEngines = UNDEFINED } // DeepCopy returns deepcopy object. diff --git a/pkg/core/typedef/nrirawpod.go b/pkg/core/typedef/nrirawpod.go index 0749c8a..d060923 100644 --- a/pkg/core/typedef/nrirawpod.go +++ b/pkg/core/typedef/nrirawpod.go @@ -49,10 +49,6 @@ const ( fileMode os.FileMode = 0666 ) -func init() { - setContainerEnginesOnce.Do(FixContainerEngine) -} - // convert NRIRawPod structure to PodInfo structure func (pod *NRIRawPod) ConvertNRIRawPod2PodInfo() *PodInfo { if pod == nil { @@ -87,7 +83,6 @@ func (pod *NRIRawPod) GetQosClass() string { // get pod cgroupPath func (pod *NRIRawPod) CgroupPath() string { - var path string id := pod.Uid qosClassPath := "" @@ -100,61 +95,7 @@ func (pod *NRIRawPod) CgroupPath() string { default: return "" } - /* - Kubernetes defines three different pods: - 1. Burstable: pod requests are less than the value of limits and not 0; - 2. BestEffort: pod requests and limits are both 0; - 3. Guaranteed: pod requests are equal to the value set by limits; - - When using cgroupfs as cgroup driver, - 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3 - 2. The BestEffort path is in the form: kubepods/besteffort/pod34152897-dbaf-11ea-8cb9-0653660051c3 - 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3 - - When using systemd as cgroup driver: - 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice - 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice - 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/ - */ - - if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join( - constant.KubepodsCgroup+".slice", - constant.KubepodsCgroup+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice", - ) - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join( - constant.KubepodsCgroup+".slice", - constant.KubepodsCgroup+"-"+qosClassPath+".slice", - pod.Linux.CgroupParent, - ) - - } - } - } else { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula, constant.ContainerEngineCrio: - path = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) - } - } else { - - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula, constant.ContainerEngineCrio: - path = filepath.Join(constant.KubepodsCgroup, qosClassPath, constant.PodCgroupNamePrefix+id) - default: - path = "" - } - } - - } - return path + return cgroup.ConcatPodCgroupPath(qosClassPath, id) } // get pod running state @@ -437,7 +378,7 @@ func (container *NRIRawContainer) GetResourceMaps() (ResourceMap, ResourceMap) { } // get current container engine -func FixContainerEngine() { +func getEngineFromCgroup() { file, err := os.OpenFile(procSelfCgroupFile, os.O_RDONLY, fileMode) if err != nil { return diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index b67126a..b653c71 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -16,13 +16,11 @@ package typedef import ( "fmt" - "path/filepath" "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/core/typedef/cgroup" ) @@ -86,83 +84,30 @@ func (pod *RawPod) ID() string { return string(pod.UID) } +// Kubernetes defines three different pods: +// 1. Burstable: pod requests are less than the value of limits and not 0; +// 2. BestEffort: pod requests and limits are both 0; +// 3. Guaranteed: pod requests are equal to the value set by limits; +var k8sQosClass = map[corev1.PodQOSClass]string{ + corev1.PodQOSGuaranteed: "", + corev1.PodQOSBurstable: strings.ToLower(string(corev1.PodQOSBurstable)), + corev1.PodQOSBestEffort: strings.ToLower(string(corev1.PodQOSBestEffort)), +} + // CgroupPath returns cgroup path of raw pod // handle different combinations of cgroupdriver and pod qos and container runtime -func (pod *RawPod) CgroupPath() string { +func (pod *RawPod) CgroupPath() (res string) { id := string(pod.UID) if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" { id = configHash } - qosClassPath := "" - switch pod.Status.QOSClass { - case corev1.PodQOSGuaranteed: - case corev1.PodQOSBurstable: - qosClassPath = strings.ToLower(string(corev1.PodQOSBurstable)) - case corev1.PodQOSBestEffort: - qosClassPath = strings.ToLower(string(corev1.PodQOSBestEffort)) - default: + qosPrefix, existed := k8sQosClass[pod.Status.QOSClass] + if !existed { + fmt.Printf("unsupported qos class: %v", pod.Status.QOSClass) return "" } - - /* - Kubernetes defines three different pods: - 1. Burstable: pod requests are less than the value of limits and not 0; - 2. BestEffort: pod requests and limits are both 0; - 3. Guaranteed: pod requests are equal to the value set by limits; - - When using cgroupfs as cgroup driver: - 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3 - 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3 - 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3 - - When using systemd as cgroup driver: - 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice - 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice - 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/ - */ - - if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - return filepath.Join( - constant.KubepodsCgroup+".slice", - constant.KubepodsCgroup+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice", - ) - default: - return "" - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - return filepath.Join( - constant.KubepodsCgroup+".slice", - constant.KubepodsCgroup+"-"+qosClassPath+".slice", - constant.KubepodsCgroup+"-"+qosClassPath+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice", - ) - default: - return "" - } - - } - } else { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineDocker, constant.ContainerEngineContainerd, constant.ContainerEngineIsula, constant.ContainerEngineCrio: - return filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) - default: - return "" - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineDocker, constant.ContainerEngineContainerd, constant.ContainerEngineIsula, constant.ContainerEngineCrio: - return filepath.Join(constant.KubepodsCgroup, qosClassPath, constant.PodCgroupNamePrefix+id) - default: - return "" - } - } - } + return cgroup.ConcatPodCgroupPath(qosPrefix, id) } // ListRawContainers returns all RawContainers in the RawPod @@ -221,6 +166,13 @@ func (cont *RawContainer) GetRealContainerID() (string, error) { So we don't consider the case of midway container engine changes `fixContainerEngine` is only executed when `getRealContainerID` is called for the first time */ + setContainerEnginesOnce.Do(func() { + getEngineFromCgroup() + _, exist := supportEnginesPrefixMap[currentContainerEngines] + if !exist { + getEngineFromContainerID(cont.status.ContainerID) + } + }) if !currentContainerEngines.Support(cont) { return "", fmt.Errorf("unsupported container engine: %v", cont.status.ContainerID) -- Gitee