diff --git a/pkg/core/typedef/cgroup/cgroupfs/driver.go b/pkg/core/typedef/cgroup/cgroupfs/driver.go new file mode 100644 index 0000000000000000000000000000000000000000..4256deb830ab1256dcf463fb66bd51fa8f0a9b3c --- /dev/null +++ b/pkg/core/typedef/cgroup/cgroupfs/driver.go @@ -0,0 +1,37 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2024-09-03 +// Description: This file is used for cgroupfs driver + +package cgroupfs + +import ( + "path/filepath" + + "isula.org/rubik/pkg/common/constant" +) + +const Name = "cgroupfs" + +type Driver struct{} + +func (d *Driver) Name() string { + return Name +} + +func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string { + // When using cgroupfs as cgroup driver: + // 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3 + // 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3 + // 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3 + + return filepath.Join(constant.KubepodsCgroup, qosClass, constant.PodCgroupNamePrefix+id) +} diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go index 668f951f33aad080fe9cec66594649fade263791..5252e7dd060f9cb819e216c03101910c8a7c8b47 100644 --- a/pkg/core/typedef/cgroup/common.go +++ b/pkg/core/typedef/cgroup/common.go @@ -27,11 +27,6 @@ import ( var rootDir = constant.DefaultCgroupRoot var cgroupDriver = constant.CgroupDriverCgroupfs -// SetCgroupDriver is the setter of global cgroup driver -func SetCgroupDriver(driver string) { - cgroupDriver = driver -} - // GetCgroupDriver is the getter of global cgroup driver func GetCgroupDriver() string { return cgroupDriver diff --git a/pkg/core/typedef/cgroup/driver.go b/pkg/core/typedef/cgroup/driver.go new file mode 100644 index 0000000000000000000000000000000000000000..b8cd4d5a7bf91ce7f272dc557fdd06779374d6bf --- /dev/null +++ b/pkg/core/typedef/cgroup/driver.go @@ -0,0 +1,48 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2024-09-03 +// Description: This file is used for cgroup driver + +package cgroup + +import ( + "isula.org/rubik/pkg/core/typedef/cgroup/cgroupfs" + "isula.org/rubik/pkg/core/typedef/cgroup/systemd" +) + +type Driver interface { + Name() string + ConcatPodCgroupPath(qosClass string, id string) string +} + +var driver Driver = &cgroupfs.Driver{} + +// SetCgroupDriver is the setter of global cgroup driver +func SetCgroupDriver(driverTyp string) { + cgroupDriver = driverTyp + switch driverTyp { + case systemd.Name: + driver = &systemd.Driver{} + case cgroupfs.Name: + driver = &cgroupfs.Driver{} + } +} + +func Type() string { + return driver.Name() +} +func ConcatPodCgroupPath(qosClass, id string) string { + return driver.ConcatPodCgroupPath(qosClass, id) +} + +func ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string { + return driver.ConcatPodCgroupPath(podCgroupPath, containerScope) +} diff --git a/pkg/core/typedef/cgroup/systemd/driver.go b/pkg/core/typedef/cgroup/systemd/driver.go new file mode 100644 index 0000000000000000000000000000000000000000..740b7ba470df167512785763aa85c6da728c9626 --- /dev/null +++ b/pkg/core/typedef/cgroup/systemd/driver.go @@ -0,0 +1,51 @@ +// Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved. +// rubik licensed under the Mulan PSL v2. +// You can use this software according to the terms and conditions of the Mulan PSL v2. +// You may obtain a copy of Mulan PSL v2 at: +// http://license.coscl.org.cn/MulanPSL2 +// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +// PURPOSE. +// See the Mulan PSL v2 for more details. +// Author: Jiaqi Yang +// Date: 2024-09-03 +// Description: This file is used for system cgroup driver + +package systemd + +import ( + "path/filepath" + "strings" + + "isula.org/rubik/pkg/common/constant" +) + +const Name = "systemd" + +type Driver struct{} + +func (d *Driver) Name() string { + return Name +} + +func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string { + // When using systemd as cgroup driver: + // 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice + // 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice + // 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/ + const suffix = ".slice" + var ( + prefix = constant.KubepodsCgroup + podPath = constant.KubepodsCgroup + suffix + ) + if qosClass != "" { + podPath = filepath.Join(podPath, constant.KubepodsCgroup+"-"+qosClass+suffix) + prefix = strings.Join([]string{prefix, qosClass}, "-") + } + return filepath.Join(podPath, + strings.Join([]string{prefix, constant.PodCgroupNamePrefix + strings.Replace(id, "-", "_", -1) + suffix}, "-")) +} + +func (d *Driver) ConcatContainerCgroupPath(podCgroupPath string, containerScope string) string { + return filepath.Join(podCgroupPath, containerScope+".scope") +} diff --git a/pkg/core/typedef/containerinfo.go b/pkg/core/typedef/containerinfo.go index 841c800ad9f1d682c0d9a62557a1dc0e494cd0f2..ec04ed8836e9b007b516366cfaa7410fed04f740 100644 --- a/pkg/core/typedef/containerinfo.go +++ b/pkg/core/typedef/containerinfo.go @@ -85,33 +85,30 @@ type ContainerInfo struct { PodSandboxId string `json:"podisandid,omitempty"` } +func containerPath(id, podCgroupPath string) string { + if cgroup.Type() == constant.CgroupDriverSystemd { + return filepath.Join(podCgroupPath, containerEngineScopes[currentContainerEngines]+"-"+id+".scope") + } + // In the case of cgroupfs, the path of crio contains a special prefix + if containerEngineScopes[currentContainerEngines] == constant.ContainerEngineCrio { + return filepath.Join(podCgroupPath, constant.ContainerEngineCrio+"-"+id) + } + return filepath.Join(podCgroupPath, id) +} + // NewContainerInfo creates a ContainerInfo instance func NewContainerInfo(id, podCgroupPath string, rawContainer *RawContainer) *ContainerInfo { - scopeName := containerEngineScopes[currentContainerEngines] requests, limits := rawContainer.GetResourceMaps() - var path string - if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join(podCgroupPath, scopeName+"-"+id+".scope") - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join(podCgroupPath, id) - case constant.ContainerEngineCrio: - path = filepath.Join(podCgroupPath, scopeName+"-"+id) - } - } return &ContainerInfo{ Name: rawContainer.status.Name, ID: id, - Hierarchy: cgroup.Hierarchy{Path: path}, + Hierarchy: cgroup.Hierarchy{Path: containerPath(id, podCgroupPath)}, RequestResources: requests, - LimitResources: limits} + LimitResources: limits, + } } -func fixContainerEngine(containerID string) { +func getEngineFromContainerID(containerID string) { for engine, prefix := range supportEnginesPrefixMap { if strings.HasPrefix(containerID, prefix) { currentContainerEngines = engine @@ -119,7 +116,6 @@ func fixContainerEngine(containerID string) { return } } - currentContainerEngines = UNDEFINED } // DeepCopy returns deepcopy object. diff --git a/pkg/core/typedef/nrirawpod.go b/pkg/core/typedef/nrirawpod.go index 0749c8a53467875e23310135e96333efcb5359dd..d06092375732e94f6924b4f8cf4e43931451110b 100644 --- a/pkg/core/typedef/nrirawpod.go +++ b/pkg/core/typedef/nrirawpod.go @@ -49,10 +49,6 @@ const ( fileMode os.FileMode = 0666 ) -func init() { - setContainerEnginesOnce.Do(FixContainerEngine) -} - // convert NRIRawPod structure to PodInfo structure func (pod *NRIRawPod) ConvertNRIRawPod2PodInfo() *PodInfo { if pod == nil { @@ -87,7 +83,6 @@ func (pod *NRIRawPod) GetQosClass() string { // get pod cgroupPath func (pod *NRIRawPod) CgroupPath() string { - var path string id := pod.Uid qosClassPath := "" @@ -100,61 +95,7 @@ func (pod *NRIRawPod) CgroupPath() string { default: return "" } - /* - Kubernetes defines three different pods: - 1. Burstable: pod requests are less than the value of limits and not 0; - 2. BestEffort: pod requests and limits are both 0; - 3. Guaranteed: pod requests are equal to the value set by limits; - - When using cgroupfs as cgroup driver, - 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3 - 2. The BestEffort path is in the form: kubepods/besteffort/pod34152897-dbaf-11ea-8cb9-0653660051c3 - 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3 - - When using systemd as cgroup driver: - 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice - 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice - 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/ - */ - - if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join( - constant.KubepodsCgroup+".slice", - constant.KubepodsCgroup+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice", - ) - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - path = filepath.Join( - constant.KubepodsCgroup+".slice", - constant.KubepodsCgroup+"-"+qosClassPath+".slice", - pod.Linux.CgroupParent, - ) - - } - } - } else { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula, constant.ContainerEngineCrio: - path = filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) - } - } else { - - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineDocker, constant.ContainerEngineIsula, constant.ContainerEngineCrio: - path = filepath.Join(constant.KubepodsCgroup, qosClassPath, constant.PodCgroupNamePrefix+id) - default: - path = "" - } - } - - } - return path + return cgroup.ConcatPodCgroupPath(qosClassPath, id) } // get pod running state @@ -437,7 +378,7 @@ func (container *NRIRawContainer) GetResourceMaps() (ResourceMap, ResourceMap) { } // get current container engine -func FixContainerEngine() { +func getEngineFromCgroup() { file, err := os.OpenFile(procSelfCgroupFile, os.O_RDONLY, fileMode) if err != nil { return diff --git a/pkg/core/typedef/rawpod.go b/pkg/core/typedef/rawpod.go index b67126aeff00ec026fb1d27c9be1a43fd2e6b744..b653c714e4af2a238558e5703e77921626910d58 100644 --- a/pkg/core/typedef/rawpod.go +++ b/pkg/core/typedef/rawpod.go @@ -16,13 +16,11 @@ package typedef import ( "fmt" - "path/filepath" "strings" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" - "isula.org/rubik/pkg/common/constant" "isula.org/rubik/pkg/core/typedef/cgroup" ) @@ -86,83 +84,30 @@ func (pod *RawPod) ID() string { return string(pod.UID) } +// Kubernetes defines three different pods: +// 1. Burstable: pod requests are less than the value of limits and not 0; +// 2. BestEffort: pod requests and limits are both 0; +// 3. Guaranteed: pod requests are equal to the value set by limits; +var k8sQosClass = map[corev1.PodQOSClass]string{ + corev1.PodQOSGuaranteed: "", + corev1.PodQOSBurstable: strings.ToLower(string(corev1.PodQOSBurstable)), + corev1.PodQOSBestEffort: strings.ToLower(string(corev1.PodQOSBestEffort)), +} + // CgroupPath returns cgroup path of raw pod // handle different combinations of cgroupdriver and pod qos and container runtime -func (pod *RawPod) CgroupPath() string { +func (pod *RawPod) CgroupPath() (res string) { id := string(pod.UID) if configHash := pod.Annotations[configHashAnnotationKey]; configHash != "" { id = configHash } - qosClassPath := "" - switch pod.Status.QOSClass { - case corev1.PodQOSGuaranteed: - case corev1.PodQOSBurstable: - qosClassPath = strings.ToLower(string(corev1.PodQOSBurstable)) - case corev1.PodQOSBestEffort: - qosClassPath = strings.ToLower(string(corev1.PodQOSBestEffort)) - default: + qosPrefix, existed := k8sQosClass[pod.Status.QOSClass] + if !existed { + fmt.Printf("unsupported qos class: %v", pod.Status.QOSClass) return "" } - - /* - Kubernetes defines three different pods: - 1. Burstable: pod requests are less than the value of limits and not 0; - 2. BestEffort: pod requests and limits are both 0; - 3. Guaranteed: pod requests are equal to the value set by limits; - - When using cgroupfs as cgroup driver: - 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3 - 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3 - 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3 - - When using systemd as cgroup driver: - 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice - 2. The BestEffort path is in the form: kubepods.slice/kubepods-besteffort.slice/kubepods-besteffort-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice - 3. The Guaranteed path is in the form: kubepods.slice/kubepods-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice/ - */ - - if cgroup.GetCgroupDriver() == constant.CgroupDriverSystemd { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - return filepath.Join( - constant.KubepodsCgroup+".slice", - constant.KubepodsCgroup+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice", - ) - default: - return "" - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineContainerd, constant.ContainerEngineCrio, constant.ContainerEngineDocker, constant.ContainerEngineIsula: - return filepath.Join( - constant.KubepodsCgroup+".slice", - constant.KubepodsCgroup+"-"+qosClassPath+".slice", - constant.KubepodsCgroup+"-"+qosClassPath+"-"+constant.PodCgroupNamePrefix+strings.Replace(id, "-", "_", -1)+".slice", - ) - default: - return "" - } - - } - } else { - if qosClassPath == "" { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineDocker, constant.ContainerEngineContainerd, constant.ContainerEngineIsula, constant.ContainerEngineCrio: - return filepath.Join(constant.KubepodsCgroup, constant.PodCgroupNamePrefix+id) - default: - return "" - } - } else { - switch containerEngineScopes[currentContainerEngines] { - case constant.ContainerEngineDocker, constant.ContainerEngineContainerd, constant.ContainerEngineIsula, constant.ContainerEngineCrio: - return filepath.Join(constant.KubepodsCgroup, qosClassPath, constant.PodCgroupNamePrefix+id) - default: - return "" - } - } - } + return cgroup.ConcatPodCgroupPath(qosPrefix, id) } // ListRawContainers returns all RawContainers in the RawPod @@ -221,6 +166,13 @@ func (cont *RawContainer) GetRealContainerID() (string, error) { So we don't consider the case of midway container engine changes `fixContainerEngine` is only executed when `getRealContainerID` is called for the first time */ + setContainerEnginesOnce.Do(func() { + getEngineFromCgroup() + _, exist := supportEnginesPrefixMap[currentContainerEngines] + if !exist { + getEngineFromContainerID(cont.status.ContainerID) + } + }) if !currentContainerEngines.Support(cont) { return "", fmt.Errorf("unsupported container engine: %v", cont.status.ContainerID)