diff --git a/pkg/core/typedef/cgroup/cgroupfs/driver.go b/pkg/core/typedef/cgroup/cgroupfs/driver.go index 87fb7480483a6707e85b2c1110c1469c5a939bcc..89eb01b9237aead5e18dbca5e66693a7028e4266 100644 --- a/pkg/core/typedef/cgroup/cgroupfs/driver.go +++ b/pkg/core/typedef/cgroup/cgroupfs/driver.go @@ -19,23 +19,27 @@ import ( "isula.org/rubik/pkg/common/constant" ) +// Name is the name of cgroupfs const Name = "cgroupfs" +// Driver is the implement of cgroupfs methods type Driver struct{} +// Name returns the name of driver func (d *Driver) Name() string { return Name } +// ConcatPodCgroupPath returns the cgroup path of pod when driver is cgroupfs func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string { // When using cgroupfs as cgroup driver: // 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3 // 2. The BestEffort path is in the form: kubepods/bestEffort/pod34152897-dbaf-11ea-8cb9-0653660051c3 // 3. The Guaranteed path is in the form: kubepods/pod34152897-dbaf-11ea-8cb9-0653660051c3 - return filepath.Join(constant.KubepodsCgroup, qosClass, constant.PodCgroupNamePrefix+id) } +// GetNRIContainerCgroupPath returns the cgroup path of nri container when driver is cgroupfs func (d *Driver) GetNRIContainerCgroupPath(nriCgroupPath string) string { // When using cgroupfs as cgroup driver and isula, docker, containerd as container runtime: // 1. The Burstable path looks like: kubepods/burstable/pod34152897-dbaf-11ea-8cb9-0653660051c3/88a791aa2090c928667579ea11a63f0ab67cf0be127743308a6e1a2130489dec @@ -44,6 +48,7 @@ func (d *Driver) GetNRIContainerCgroupPath(nriCgroupPath string) string { return nriCgroupPath } +// ConcatContainerCgroup returns the cgroup path of container from kubernetes apiserver when driver is cgroupfs func (d *Driver) ConcatContainerCgroup(podCgroupPath, prefix, containerID string) string { if prefix != "" { prefix = prefix + "-" diff --git a/pkg/core/typedef/cgroup/common.go b/pkg/core/typedef/cgroup/common.go index 5252e7dd060f9cb819e216c03101910c8a7c8b47..9e402f26a939e0359865025390936374db293e19 100644 --- a/pkg/core/typedef/cgroup/common.go +++ b/pkg/core/typedef/cgroup/common.go @@ -24,17 +24,50 @@ import ( "isula.org/rubik/pkg/common/util" ) -var rootDir = constant.DefaultCgroupRoot -var cgroupDriver = constant.CgroupDriverCgroupfs +// Config is the configuration of cgroup +type Config struct { + RootDir string + CgroupDriver Driver +} + +var conf = &Config{ + RootDir: constant.DefaultCgroupRoot, + CgroupDriver: defaultDriver(), +} -// GetCgroupDriver is the getter of global cgroup driver -func GetCgroupDriver() string { - return cgroupDriver +type option func(c *Config) error + +func WithRoot(cgroupRoot string) option { + return func(c *Config) error { + c.RootDir = cgroupRoot + return nil + } +} + +func WithDriver(driverType string) option { + return func(c *Config) error { + d, err := newCgroupDriver(driverType) + if err != nil { + return err + } + c.CgroupDriver = d + return nil + } +} + +// Init sets the mount directory of the cgroup file system & driver +func Init(opts ...option) error { + for _, opt := range opts { + if err := opt(conf); err != nil { + return fmt.Errorf("failed to init cgroup: %v", err) + } + } + return nil } // AbsoluteCgroupPath returns the absolute path of the cgroup func AbsoluteCgroupPath(elem ...string) string { - elem = append([]string{rootDir}, elem...) + elem = append([]string{conf.RootDir}, elem...) return filepath.Join(elem...) } @@ -62,14 +95,9 @@ func writeCgroupFile(cgPath, content string) error { return util.WriteFile(cgPath, content) } -// InitMountDir sets the mount directory of the cgroup file system -func InitMountDir(arg string) { - rootDir = arg -} - // GetMountDir returns the mount point path of the cgroup func GetMountDir() string { - return rootDir + return conf.RootDir } type ( @@ -184,7 +212,7 @@ func (h *Hierarchy) SetCgroupAttr(key *Key, value string) error { if err := validateCgroupKey(key); err != nil { return err } - var mountPoint = rootDir + var mountPoint = conf.RootDir if len(h.MountPoint) > 0 { mountPoint = h.MountPoint } @@ -196,7 +224,7 @@ func (h *Hierarchy) GetCgroupAttr(key *Key) *Attr { if err := validateCgroupKey(key); err != nil { return &Attr{Err: err} } - var mountPoint = rootDir + var mountPoint = conf.RootDir if len(h.MountPoint) > 0 { mountPoint = h.MountPoint } diff --git a/pkg/core/typedef/cgroup/common_test.go b/pkg/core/typedef/cgroup/common_test.go index b7556d6be44dac154103e648dc9b059b3a98d43b..829a6b617e4d665c98dd50c656af4ddfaf3dcd16 100644 --- a/pkg/core/typedef/cgroup/common_test.go +++ b/pkg/core/typedef/cgroup/common_test.go @@ -24,13 +24,24 @@ import ( "github.com/stretchr/testify/assert" "isula.org/rubik/pkg/common/constant" + "isula.org/rubik/pkg/common/log" "isula.org/rubik/pkg/common/util" ) // TestReadCgroupFile tests ReadCgroupFile func TestReadCgroupFile(t *testing.T) { - InitMountDir(constant.TmpTestDir) - defer InitMountDir(constant.DefaultCgroupRoot) + if err := os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode); err != nil { + log.Infof("failed to create %v: %v", constant.TmpTestDir, err) + return + } + defer os.RemoveAll(constant.TmpTestDir) + + if err := Init(WithRoot(constant.TmpTestDir)); err != nil { + log.Infof("failed to init cgroup: %v", err) + return + } + defer Init(WithRoot(constant.DefaultCgroupRoot)) + pathElems := []string{"cpu", "kubepods", "PodXXX", "ContYYY", "cpu.cfs_quota_us"} const value = "-1\n" tests := []struct { @@ -84,8 +95,17 @@ func TestReadCgroupFile(t *testing.T) { // TestWriteCgroupFile tests WriteCgroupFile func TestWriteCgroupFile(t *testing.T) { - InitMountDir(constant.TmpTestDir) - defer InitMountDir(constant.DefaultCgroupRoot) + if err := os.Mkdir(constant.TmpTestDir, constant.DefaultDirMode); err != nil { + log.Infof("failed to create %v: %v", constant.TmpTestDir, err) + return + } + defer os.RemoveAll(constant.TmpTestDir) + + if err := Init(WithRoot(constant.TmpTestDir)); err != nil { + log.Infof("failed to init cgroup: %v", err) + return + } + defer Init(WithRoot(constant.DefaultCgroupRoot)) pathElems := []string{"cpu", "kubepods", "PodXXX", "ContYYY", "cpu.cfs_quota_us"} const value = "-1\n" type args struct { diff --git a/pkg/core/typedef/cgroup/driver.go b/pkg/core/typedef/cgroup/driver.go index 4d5225b05ab9f4a306a16372fc217893cccc1007..e5aa1ba7fc4eb1cb5605a256e7a506358a706a92 100644 --- a/pkg/core/typedef/cgroup/driver.go +++ b/pkg/core/typedef/cgroup/driver.go @@ -14,10 +14,13 @@ package cgroup import ( + "fmt" + "isula.org/rubik/pkg/core/typedef/cgroup/cgroupfs" "isula.org/rubik/pkg/core/typedef/cgroup/systemd" ) +// Driver is the interface of cgroup methods type Driver interface { Name() string ConcatPodCgroupPath(qosClass string, id string) string @@ -25,30 +28,37 @@ type Driver interface { GetNRIContainerCgroupPath(nriCgroupPath string) string } -var driver Driver = &cgroupfs.Driver{} +func defaultDriver() Driver { + return &cgroupfs.Driver{} +} -// SetCgroupDriver is the setter of global cgroup driver -func SetCgroupDriver(driverTyp string) { - cgroupDriver = driverTyp +// NewCgroupDriver is the setter of global cgroup driver, only support systemd & cgroupfs +func newCgroupDriver(driverTyp string) (Driver, error) { switch driverTyp { case systemd.Name: - driver = &systemd.Driver{} + return &systemd.Driver{}, nil case cgroupfs.Name: - driver = &cgroupfs.Driver{} + return &cgroupfs.Driver{}, nil } + return nil, fmt.Errorf("invalid driver type: %v", driverTyp) } +// Type returns the driver type func Type() string { - return driver.Name() + return conf.CgroupDriver.Name() } + +// ConcatPodCgroupPath returns the cgroup path of pod func ConcatPodCgroupPath(qosClass, id string) string { - return driver.ConcatPodCgroupPath(qosClass, id) + return conf.CgroupDriver.ConcatPodCgroupPath(qosClass, id) } +// GetNRIContainerCgroupPath returns the cgroup path of nri container func GetNRIContainerCgroupPath(nriCgroupPath string) string { - return driver.GetNRIContainerCgroupPath(nriCgroupPath) + return conf.CgroupDriver.GetNRIContainerCgroupPath(nriCgroupPath) } +// ConcatContainerCgroup returns the cgroup path of container from kubernetes apiserver func ConcatContainerCgroup(podCgroupPath, prefix, containerID string) string { - return driver.ConcatContainerCgroup(podCgroupPath, prefix, containerID) + return conf.CgroupDriver.ConcatContainerCgroup(podCgroupPath, prefix, containerID) } diff --git a/pkg/core/typedef/cgroup/systemd/driver.go b/pkg/core/typedef/cgroup/systemd/driver.go index de8f956d5b7e481cc8e6c1787f37a033dff000b5..3635c2174e5e7f39f09e7904d77656fd6ef84b8d 100644 --- a/pkg/core/typedef/cgroup/systemd/driver.go +++ b/pkg/core/typedef/cgroup/systemd/driver.go @@ -21,16 +21,21 @@ import ( ) const ( - Name = "systemd" + // Name is the name of systemd + Name = "systemd" + // suffix of systemd cgroup file cgroupFileExt = ".scope" ) +// Driver is the implement of systemd methods type Driver struct{} +// Name returns the name of driver func (d *Driver) Name() string { return Name } +// ConcatPodCgroupPath returns the cgroup path of pod when driver is systemd func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string { // When using systemd as cgroup driver: // 1. The Burstable path looks like: kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podb895995a_e7e5_413e_9bc1_3c3895b3f233.slice @@ -49,6 +54,7 @@ func (d *Driver) ConcatPodCgroupPath(qosClass string, id string) string { strings.Join([]string{prefix, constant.PodCgroupNamePrefix + strings.Replace(id, "-", "_", -1) + suffix}, "-")) } +// GetNRIContainerCgroupPath returns the cgroup path of nri container when driver is systemd func (d *Driver) GetNRIContainerCgroupPath(nriCgroupPath string) string { // When using systemd as cgroup driver: // 1. The Burstable path looks like: @@ -84,6 +90,7 @@ func (d *Driver) GetNRIContainerCgroupPath(nriCgroupPath string) string { return filepath.Join(topDir, upperDir, parent, last) } +// ConcatContainerCgroup returns the cgroup path of container from kubernetes apiserver when driver is systemd func (d *Driver) ConcatContainerCgroup(podCgroupPath, prefix, containerID string) string { if prefix != "" { prefix = prefix + "-" diff --git a/pkg/informer/informerfactory.go b/pkg/informer/informerfactory.go index 9d2408007c5c2adeb6eb9d13d6954d634f69aced..203aeb5fabd86e41860a27c73dc71ed66d8425e9 100644 --- a/pkg/informer/informerfactory.go +++ b/pkg/informer/informerfactory.go @@ -21,25 +21,22 @@ import ( ) type ( - // the definition of informer type - informerType int8 // informer's factory class informerFactory struct{} informerCreator func(publisher api.Publisher) (api.Informer, error) ) const ( - // APISERVER instructs the informer to interact with the api server of kubernetes to obtain data - APISERVER informerType = iota - NRI + APISERVER = "apiserver" // the informer to interact with the apiserver of kubernetes + NRI = "nri" // the informer to interact with the NRI interface ) // defaultInformerFactory is globally unique informer factory var defaultInformerFactory *informerFactory // GetInformerCreator returns the constructor of the informer of the specified type -func (factory *informerFactory) GetInformerCreator(iType informerType) informerCreator { - switch iType { +func (factory *informerFactory) GetInformerCreator(informerName string) informerCreator { + switch informerName { case APISERVER: return NewAPIServerInformer case NRI: diff --git a/pkg/informer/nriinformer.go b/pkg/informer/nriinformer.go index e3af51d493fe2d5d87fe524e2799d10c43b6a2d7..1d6f2088df7e2f9815dcea4322febda0fef25f70 100644 --- a/pkg/informer/nriinformer.go +++ b/pkg/informer/nriinformer.go @@ -27,7 +27,12 @@ import ( "isula.org/rubik/pkg/core/typedef" ) -// NRIInformer interacts with crio server and forward data to the internal +const ( + nriPluginName = "rubik" + nriPluginIndex = "10" +) + +// NRIInformer interacts with nri server and forward data to the internal type NRIInformer struct { rubikapi.Publisher nodeName string @@ -35,25 +40,27 @@ type NRIInformer struct { finishedSync chan struct{} } -// register nriplugin +// NewNRIInformer create an rubik nri plugin func NewNRIInformer(publisher rubikapi.Publisher) (rubikapi.Informer, error) { - p := &NRIInformer{} - p.Publisher = publisher - pluginName := "rubik" - pluginIndex := "10" - nodeName := os.Getenv(constant.NodeNameEnvKey) - p.finishedSync = make(chan struct{}) - p.nodeName = nodeName - var err error + p := &NRIInformer{ + Publisher: publisher, + nodeName: os.Getenv(constant.NodeNameEnvKey), + finishedSync: make(chan struct{}), + } + options := []stub.Option{ - stub.WithPluginName(pluginName), - stub.WithPluginIdx(pluginIndex), + stub.WithPluginName(nriPluginName), + stub.WithPluginIdx(nriPluginIndex), + } + s, err := stub.New(p, options...) + if err != nil { + return nil, fmt.Errorf("failed to create stub: %v", err) } - p.stub, err = stub.New(p, options...) + p.stub = s return p, err } -// start nriplugin +// Start starts nri informer func (plugin NRIInformer) Start(ctx context.Context) error { if err := plugin.stub.Start(ctx); err != nil { return fmt.Errorf("failed to start nri informer: %v", err) @@ -67,87 +74,77 @@ func (plugin NRIInformer) Start(ctx context.Context) error { return nil } -// nri sync event -func (plugin NRIInformer) Synchronize(ctx context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) { +// Synchronize syncs the nri containers & sandboxes +func (plugin NRIInformer) Synchronize(ctx context.Context, pods []*api.PodSandbox, containers []*api.Container) ( + []*api.ContainerUpdate, error) { plugin.Publish(typedef.NRIPODSYNCALL, pods) plugin.Publish(typedef.NRICONTAINERSYNCALL, containers) // notify service handler to start close(plugin.finishedSync) - return nil, nil } -// nri pod run event +// RunPodSandbox will be called when sandbox starts. func (plugin NRIInformer) RunPodSandbox(ctx context.Context, pod *api.PodSandbox) error { plugin.Publish(typedef.NRIPODADD, pod) return nil } -// nri pod stop event +// StopPodSandbox will be called when sandbox stops. func (plugin NRIInformer) StopPodSandbox(ctx context.Context, pod *api.PodSandbox) error { return nil } -// nri pod remove event +// RemovePodSandbox will be called when sandbox is removed. func (plugin NRIInformer) RemovePodSandbox(ctx context.Context, pod *api.PodSandbox) error { plugin.Publish(typedef.NRIPODDELETE, pod) - return nil } -// nri container create event -func (plugin NRIInformer) CreateContainer(context.Context, *api.PodSandbox, *api.Container) (*api.ContainerAdjustment, []*api.ContainerUpdate, error) { - var containerAdjustment = &api.ContainerAdjustment{} - var containerUpdate = []*api.ContainerUpdate{} - return containerAdjustment, containerUpdate, nil +// CreateContainer will be called when it creates container +func (plugin NRIInformer) CreateContainer(context.Context, *api.PodSandbox, *api.Container) ( + *api.ContainerAdjustment, []*api.ContainerUpdate, error) { + return nil, nil, nil } -// nri container start event +// StartContainer will be called when container starts func (plugin NRIInformer) StartContainer(ctx context.Context, pod *api.PodSandbox, container *api.Container) error { plugin.Publish(typedef.NRICONTAINERSTART, container) return nil } -// nri container update event +// UpdateContainer will be called when container updates func (plugin NRIInformer) UpdateContainer(ctx context.Context, pod *api.PodSandbox, container *api.Container, lr *api.LinuxResources) ([]*api.ContainerUpdate, error) { - var containerUpdate = []*api.ContainerUpdate{} - return containerUpdate, nil + return nil, nil } -// nri container stop cont +// StopContainer will be called when container stops func (plugin NRIInformer) StopContainer(ctx context.Context, pod *api.PodSandbox, container *api.Container) ([]*api.ContainerUpdate, error) { plugin.Publish(typedef.NRICONTAINERREMOVE, container) - var containerUpdate = []*api.ContainerUpdate{} - return containerUpdate, nil + return nil, nil } -// nri container remove event +// RemoveContainer will be called when it removes container func (plugin NRIInformer) RemoveContainer(ctx context.Context, pod *api.PodSandbox, container *api.Container) error { plugin.Publish(typedef.NRICONTAINERREMOVE, container) return nil } -// nri configure event -func (plugin NRIInformer) Configure(context.Context, string, string, string) (api.EventMask, error) { - var eventMask api.EventMask - return eventMask, nil -} - -// nri post container create event +// PostCreateContainer will be called after container was created func (plugin NRIInformer) PostCreateContainer(context.Context, *api.PodSandbox, *api.Container) error { return nil } -// nri post container start event +// ostStartContainer will be called after container was started func (plugin NRIInformer) PostStartContainer(context.Context, *api.PodSandbox, *api.Container) error { return nil } -// nri post container update event +// PostUpdateContainer will be called after container was updated func (plugin NRIInformer) PostUpdateContainer(context.Context, *api.PodSandbox, *api.Container) error { return nil } -// nri shutdown event +// Shutdown will be called when nri plugin shutdowns func (plugin NRIInformer) Shutdown(context.Context) { } diff --git a/pkg/rubik/rubik.go b/pkg/rubik/rubik.go index c4026998a3c5b5e1f0475471c377df44567609e0..ddc7ff267a7a03496ec1ea7623b1ef12820fbde2 100644 --- a/pkg/rubik/rubik.go +++ b/pkg/rubik/rubik.go @@ -63,33 +63,22 @@ func NewAgent(cfg *config.Config) (*Agent, error) { // Run starts and runs the agent until receiving stop signal func (a *Agent) Run(ctx context.Context) error { log.Infof("agent run with config:\n%s", a.config.String()) - var informerName string - informerName = a.config.Agent.InformerType - if informerName == "nil" { - informerName = constant.APIServerInformer - } - if err := a.startInformer(ctx, informerName); err != nil { + if err := a.startInformer(ctx, a.config.Agent.InformerType); err != nil { return err } + defer a.stopInformer() if err := a.startServiceHandler(ctx); err != nil { return err } + defer a.stopServiceHandler() <-ctx.Done() - a.stopInformer() - a.stopServiceHandler() return nil } // startInformer starts informer to obtain external data func (a *Agent) startInformer(ctx context.Context, informerName string) error { - publisher := publisher.GetPublisherFactory().GetPublisher(publisher.GENERIC) - var i api.Informer - var err error - if informerName == constant.APIServerInformer { - i, err = informer.GetInformerFactory().GetInformerCreator(informer.APISERVER)(publisher) - } else { - i, err = informer.GetInformerFactory().GetInformerCreator(informer.NRI)(publisher) - } + i, err := informer.GetInformerFactory().GetInformerCreator(informerName)( + publisher.GetPublisherFactory().GetPublisher(publisher.GENERIC)) if err != nil { return fmt.Errorf("failed to set informer: %v", err) } @@ -135,9 +124,8 @@ func runAgent(ctx context.Context) error { } // 3. enable cgroup system - cgroup.InitMountDir(c.Agent.CgroupRoot) - if c.Agent.CgroupDriver != "" { - cgroup.SetCgroupDriver(c.Agent.CgroupDriver) + if err := cgroup.Init(cgroup.WithRoot(c.Agent.CgroupRoot), cgroup.WithDriver(c.Agent.CgroupDriver)); err != nil { + return err } // 4. init service components diff --git a/pkg/services/iocost/iocost_test.go b/pkg/services/iocost/iocost_test.go index 393bc389a7a386507aa313f6a49797fd21c20eb2..f22fa9d00c567706baaca45b566ec462e63d06ea 100644 --- a/pkg/services/iocost/iocost_test.go +++ b/pkg/services/iocost/iocost_test.go @@ -85,9 +85,9 @@ func createIOCostConfigTestItems() { func TestIOCostSupport(t *testing.T) { assert.Equal(t, ioCostSupport(), true) - cgroup.InitMountDir("/var/tmp/rubik") + cgroup.Init(cgroup.WithRoot("/var/tmp/rubik")) assert.Equal(t, ioCostSupport(), false) - cgroup.InitMountDir(constant.DefaultCgroupRoot) + cgroup.Init(cgroup.WithRoot(constant.DefaultCgroupRoot)) } func TestIOCostID(t *testing.T) { diff --git a/pkg/services/quotaburst/quotaburst_test.go b/pkg/services/quotaburst/quotaburst_test.go index 1994616ce7bd1a86285260ef5bc57a9fc16259d9..85aaba9b89f0b12c5eaceb0e93f54ef08e704c26 100644 --- a/pkg/services/quotaburst/quotaburst_test.go +++ b/pkg/services/quotaburst/quotaburst_test.go @@ -166,7 +166,7 @@ func TestBurst_AddPod(t *testing.T) { tt.args.pod.CleanPath().OrDie() }) } - cgroup.InitMountDir(constant.DefaultCgroupRoot) + cgroup.Init(cgroup.WithRoot(constant.DefaultCgroupRoot)) } // TestOther tests other function diff --git a/tests/try/pod.go b/tests/try/pod.go index 8053c4bffd267d9bb1ce4e1dadc082daf3e112c5..1cafde33996eefcaed3d4c7aa9bb57068414e82d 100644 --- a/tests/try/pod.go +++ b/tests/try/pod.go @@ -15,6 +15,7 @@ package try import ( "fmt" + "os" "path/filepath" "strings" @@ -78,7 +79,12 @@ func (pod *FakePod) genFakePodCgroupPath() Ret { if !util.PathExist(TestRoot) { MkdirAll(TestRoot, constant.DefaultDirMode).OrDie() } - cgroup.InitMountDir(pod.CGRoot) + if err := os.MkdirAll(pod.CGRoot, constant.DefaultDirMode); err != nil { + return newRet(err) + } + if err := cgroup.Init(cgroup.WithRoot(pod.CGRoot)); err != nil { + return newRet(err) + } // generate fake cgroup path for key, value := range pod.Keys { // generate pod absolute cgroup path