diff --git a/Makefile b/Makefile index fbabda6cf26eca7eef41ad89b471ebe08c6f78bc..eddf9e6d2ac952232d228d753f1b47e13f99dcbc 100644 --- a/Makefile +++ b/Makefile @@ -133,14 +133,14 @@ kustomize: $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) ARCH := $(shell uname -m) -TEST_CMD := go test ./... -race -count=1 -timeout=300s -cover -gcflags=all=-l +TEST_CMD := go test `go list ./cmd/... | grep -E 'server|controllers'` -race -count=1 -timeout=300s -cover -gcflags=all=-l -p 1 ifeq ($(ARCH), aarch64) TEST_CMD := ETCD_UNSUPPORTED_ARCH=arm64 $(TEST_CMD) endif .PHONY: test -test: manifests fmt vet envtest ## Run tests. +test: fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(TEST_CMD) .PHONY: envtest diff --git a/VERSION b/VERSION index 7dea76edb3dc51b6e5e8223e9f941a35c1e364d6..ee90284c27f187a315f1267b063fa81b5b84f613 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.1 +1.0.4 diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go index f9474b721a662a824171ab5fb504c2303c693d5a..d3d636de893826318d96d52a0a77901f68006ae3 100644 --- a/api/v1alpha1/os_types.go +++ b/api/v1alpha1/os_types.go @@ -38,6 +38,8 @@ type OSSpec struct { SysConfigs SysConfigs `json:"sysconfigs"` // +kubebuilder:validation:Optional UpgradeConfigs SysConfigs `json:"upgradeconfigs"` + // +kubebuilder:validation:Optional + NodeSelector string `json:"nodeselector"` } // +kubebuilder:subresource:status diff --git a/cmd/admin-container/main.go b/cmd/admin-container/main.go index f6a72939f20317f94de7c1b806dae603ef53c39f..5fa083815f3c15ac95b0a8ef3361d89427461b0f 100644 --- a/cmd/admin-container/main.go +++ b/cmd/admin-container/main.go @@ -22,28 +22,42 @@ import ( "github.com/sirupsen/logrus" ) +const ( + bashPath = "/usr/bin/bash" + usrBin = "/usr/bin" + usrSbin = "/usr/sbin" + localBin = "/usr/local/bin" + localSbin = "/usr/local/sbin" + usrLib = "/usr/lib" + usrLib64 = "/usr/lib64" + lib = "/lib" + lib64 = "/lib64" + envPathPrefix = "PATH=$PATH:" + envLdLibrarPathPrefix = "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:" +) + func main() { EUID := os.Geteuid() rootEUID := 0 // 0 indicates that the process has the permission of the root user. if EUID != rootEUID { logrus.Error("please use root to run hostshell") - return + } PPID := os.Getppid() rootFsPath := "/proc/" + strconv.Itoa(PPID) + "/root" - bashPath := "/usr/bin/bash" - usrBin := "/usr/bin" - usrSbin := "/usr/sbin" - localBin := "/usr/local/bin" - localSbin := "/usr/local/sbin" - paths := []string{usrBin, usrSbin, localBin, localSbin} - for i, p := range paths { - paths[i] = rootFsPath + p - } - path := "PATH=$PATH:" + strings.Join(paths, ":") - lib := "LD_LIBRARY_PATH=/lib:/lib64:/usr/lib:/usr/lib64:$LD_LIBRARY_PATH" + path := concatenateEnvPath(rootFsPath, envPathPrefix, []string{usrBin, usrSbin, localBin, localSbin}) + libPath := concatenateEnvPath(rootFsPath, envLdLibrarPathPrefix, []string{usrLib, usrLib64, lib, lib64}) if err := syscall.Exec("/usr/bin/nsenter", []string{"nsenter", "-t", "1", "-a", - "env", "-i", path, lib, rootFsPath + bashPath}, os.Environ()); err != nil { + "env", "-i", path, libPath, rootFsPath + bashPath}, os.Environ()); err != nil { logrus.Error("nsenter excute error", err) } } + +func concatenateEnvPath(prefix string, envVarPrefix string, paths []string) string { + for i, p := range paths { + paths[i] = prefix + p + } + pathLine := envVarPrefix + strings.Join(paths, ":") + pathEnv := os.ExpandEnv(pathLine) + return pathEnv +} diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 67c7e2daf8fede21f8f65316c3ba5a036efc348c..7b1ed3daad60e4713efe783a648df9fc0b532778 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -13,8 +13,6 @@ package main import ( - "fmt" - "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -24,7 +22,7 @@ import ( ) func main() { - fmt.Println("Version is:", version.Version) + logrus.Infoln("Version is:", version.Version) l, err := server.NewListener(server.SockDir, server.SockName) if err != nil { logrus.Errorln("listen error" + err.Error()) diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index e7110f8ac946b64c30d4ee03224fa9953d8c210c..ee4297ab889359f7d7d4b63a93d6956336645878 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -57,7 +57,7 @@ func (k KernelSysctl) SetConfig(config *agent.SysConfig) error { } logrus.Infof("Configured kernel.sysctl %s=%s", key, keyInfo.Value) } else { - logrus.Errorf("Failed to parse kernel.sysctl config operation %s value %s", keyInfo.Operation, keyInfo.Value) + logrus.Warnf("Failed to parse kernel.sysctl, key: %s, value: %s, operation: %s", key, keyInfo.Value, keyInfo.Operation) } } return nil @@ -97,7 +97,11 @@ type GrubCmdline struct { // SetConfig sets grub.cmdline configuration func (g GrubCmdline) SetConfig(config *agent.SysConfig) error { - logrus.Info("start set grub.cmdline configuration") + if g.isCurPartition { + logrus.Info("start set grub.cmdline.current configuration") + } else { + logrus.Info("start set grub.cmdline.next configuration") + } fileExist, err := checkFileExist(getGrubCfgPath()) if err != nil { logrus.Errorf("Failed to find config path: %v", err) @@ -259,7 +263,7 @@ func getAndSetConfigsFromFile(expectConfigs map[string]*agent.KeyInfo, path stri configsWrite = append(configsWrite, line) continue } - configKV := strings.Split(line, "=") + configKV := strings.SplitN(line, "=", kvPair) if len(configKV) != kvPair { logrus.Errorf("could not parse systctl config %s", line) return nil, fmt.Errorf("could not parse systctl config %s", line) @@ -301,6 +305,9 @@ func writeConfigToFile(path string, configs []string) error { if err = w.Flush(); err != nil { return err } + if err := f.Sync(); err != nil { + return err + } return nil } @@ -317,7 +324,11 @@ func createConfigPath(configPath string) error { if err != nil { return err } - defer f.Close() + err = f.Chmod(defaultKernelConPermission) + if err != nil { + return err + } + f.Close() return nil } @@ -335,11 +346,16 @@ func getGrubCfgPath() string { // handleDeleteKey deletes key if oldValue==newValue and returns "" string. Otherwier, it returns key=oldValue func handleDeleteKey(config []string, configInfo *agent.KeyInfo) string { - if len(config) == onlyKey { - logrus.Infoln("delete configuration ", config[0]) + key := config[0] + if len(config) == onlyKey && configInfo.Value == "" { + logrus.Infoln("delete configuration", key) return "" + } else if len(config) == onlyKey && configInfo.Value != "" { + logrus.Warnf("Failed to delete key %s with inconsistent values "+ + "nil and %s", key, configInfo.Value) + return key } - key, oldValue := config[0], config[1] + oldValue := config[1] if oldValue != configInfo.Value { logrus.Warnf("Failed to delete key %s with inconsistent values "+ "%s and %s", key, oldValue, configInfo.Value) @@ -351,22 +367,30 @@ func handleDeleteKey(config []string, configInfo *agent.KeyInfo) string { // handleUpdateKey updates key if key is found, otherwise it returns old config. func handleUpdateKey(config []string, configInfo *agent.KeyInfo, isFound bool) string { - if len(config) == onlyKey { - return config[0] + key := config[0] + if !isFound && len(config) == onlyKey { + return key } - key, oldValue := config[0], config[1] - if !isFound { - return key + "=" + oldValue + if !isFound && len(config) == kvPair { + return key + "=" + config[1] } if configInfo.Operation != "" { logrus.Warnf("Unknown operation %s, updating key %s with value %s by default", configInfo.Operation, key, configInfo.Value) } + if len(config) == onlyKey && configInfo.Value == "" { + return key + } + newValue := strings.TrimSpace(configInfo.Value) + if len(config) == onlyKey && configInfo.Value != "" { + logrus.Infof("update configuration %s=%s", key, newValue) + return key + "=" + newValue + } + oldValue := config[1] if configInfo.Value == "" { logrus.Warnf("Failed to update key %s with null value", key) return key + "=" + oldValue } - newValue := strings.TrimSpace(configInfo.Value) logrus.Infof("update configuration %s=%s", key, newValue) return key + "=" + newValue } @@ -378,9 +402,17 @@ func handleAddKey(m map[string]*agent.KeyInfo, isOnlyKeyValid bool) []string { logrus.Warnf("Failed to delete inexistent key %s", key) continue } + if key == "" || strings.Contains(key, "=") { + logrus.Warnf("Failed to add nil key or key containing =, key: %s", key) + continue + } + if keyInfo.Operation != "" { + logrus.Warnf("Unknown operation %s, adding key %s with value %s by default", + keyInfo.Operation, key, keyInfo.Value) + } k, v := strings.TrimSpace(key), strings.TrimSpace(keyInfo.Value) if keyInfo.Value == "" && isOnlyKeyValid { - logrus.Infoln("add configuration ", k) + logrus.Infoln("add configuration", k) configs = append(configs, k) } else if keyInfo.Value == "" { logrus.Warnf("Failed to add key %s with null value", k) diff --git a/cmd/agent/server/config_test.go b/cmd/agent/server/config_test.go index 903af87871e90e3da381c3d408fa94fac65b2de0..29bb9268ad8d03726ad166b31c9918c337f77631 100644 --- a/cmd/agent/server/config_test.go +++ b/cmd/agent/server/config_test.go @@ -23,7 +23,6 @@ import ( "testing" "github.com/agiledragon/gomonkey/v2" - agent "openeuler.org/KubeOS/cmd/agent/api" ) @@ -67,6 +66,25 @@ func TestKernelSysctl_SetConfig(t *testing.T) { }, }}, }, + { + name: "nil value", + k: KernelSysctl{}, + args: args{config: &agent.SysConfig{ + Contents: map[string]*agent.KeyInfo{ + "d": {Value: ""}, + }, + }}, + }, + { + name: "nil key", + k: KernelSysctl{}, + args: args{config: &agent.SysConfig{ + Contents: map[string]*agent.KeyInfo{ + "": {Value: "1"}, + }, + }}, + wantErr: true, + }, } tmpDir := t.TempDir() patchGetProcPath := gomonkey.ApplyFuncReturn(getDefaultProcPath, tmpDir+"/") @@ -84,6 +102,7 @@ func TestKernelSysctl_SetConfig(t *testing.T) { func TestKerSysctlPersist_SetConfig(t *testing.T) { tmpDir := t.TempDir() persistPath := tmpDir + "/test-persist.conf" + comment := `# This file is managed by KubeOS for unit testing.` type args struct { config *agent.SysConfig } @@ -94,21 +113,36 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { want []string wantErr bool }{ + {name: "create file", args: args{config: &agent.SysConfig{ConfigPath: persistPath}}, want: []string{comment}, wantErr: false}, + { + name: "nil path", + args: args{ + config: &agent.SysConfig{}, + }, + want: []string{}, + wantErr: false, + }, { name: "add configs", args: args{ config: &agent.SysConfig{ ConfigPath: persistPath, Contents: map[string]*agent.KeyInfo{ - "a": {Value: "1"}, - "b": {Value: "2"}, - "c": {Value: ""}, + "a": {Value: "1"}, + "b": {Value: "2"}, + "c": {Value: ""}, + "": {Value: "4"}, + "e": {Value: "5", Operation: "xxx"}, + "y=1": {Value: "26"}, + "z": {Value: "x=1"}, }, }, }, want: []string{ "a=1", "b=2", + "e=5", + "z=x=1", }, wantErr: false, }, @@ -120,12 +154,15 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { Contents: map[string]*agent.KeyInfo{ "a": {Value: "2"}, "b": {Value: ""}, + "z": {Value: "x=2", Operation: "zzz"}, }, }, }, want: []string{ "a=2", "b=2", + "e=5", + "z=x=2", }, wantErr: false, }, @@ -137,29 +174,44 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { Contents: map[string]*agent.KeyInfo{ "a": {Value: "1", Operation: "delete"}, "b": {Value: "2", Operation: "delete"}, + "c": {Value: "3", Operation: "delete"}, + "e": {Value: "5", Operation: "remove"}, + "f": {Value: "6", Operation: "remove"}, + "z": {Value: "x=2", Operation: "delete"}, }, }, }, want: []string{ "a=2", + "e=5", + "f=6", }, wantErr: false, }, } + patchGetKernelConPath := gomonkey.ApplyFuncReturn(getKernelConPath, persistPath) + defer patchGetKernelConPath.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { k := KerSysctlPersist{} if err := k.SetConfig(tt.args.config); (err != nil) != tt.wantErr { t.Errorf("KerSysctlPersist.SetConfig() error = %v, wantErr %v", err, tt.wantErr) } + if tt.name == "create file" { + if err := os.WriteFile(persistPath, []byte(comment), 0644); err != nil { + t.Fatalf("failed to write file %s", persistPath) + } + } data, err := os.ReadFile(persistPath) if err != nil { t.Errorf("failed to read file %s", persistPath) } lines := strings.Split(string(data), "\n") - // remove the last empty line - lines = lines[:len(lines)-1] - sort.Strings(lines) + if tt.name != "create file" { + // remove the comment and the last empty line + lines = lines[1 : len(lines)-1] + sort.Strings(lines) + } if !reflect.DeepEqual(lines, tt.want) { t.Errorf("KerSysctlPersist file contents not equal, expect: %v, get: %v", tt.want, lines) } @@ -210,17 +262,38 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri args: args{ config: &agent.SysConfig{ Contents: map[string]*agent.KeyInfo{ - "panic": {Value: "5"}, // update existent kv - "quiet": {Value: "", Operation: "delete"}, // delete existent key - "oops": {Value: ""}, // update existent kv with null value - "selinux": {Value: "1", Operation: "delete"}, // failed to delete inconsistent kv - "acpi": {Value: "off", Operation: "delete"}, // failed to delete inexistent kv - "debug": {}, // add key - "pci": {Value: "nomis"}, // add kv + "debug": {}, // add key + "pci": {Value: "nomis"}, // add kv + "quiet": {Value: "", Operation: "delete"}, // delete existent key + "panic": {Value: "5"}, // update existent kv + "nomodeset": {Operation: "update"}, // invalid operation, default to update existent key + "softlockup_panic": {Value: "0", Operation: "update"}, // invalid operation, default to update existent kv + "oops": {Value: ""}, // warning, skip, update existent kv with null value + "": {Value: "test"}, // warning, skip, failed to add kv with empty key + "selinux": {Value: "1", Operation: "delete"}, // failed to delete inconsistent kv + "acpi": {Value: "off", Operation: "delete"}, // failed to delete inexistent kv + "ro": {Value: "1"}, // update key to kv }, }, }, - pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=1\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro=1\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + wantErr: false, + }, + { + name: "delete and invalid operation", + g: GrubCmdline{isCurPartition: true}, + args: args{ + config: &agent.SysConfig{ + Contents: map[string]*agent.KeyInfo{ + "debug": {Operation: "delete"}, // delete key + "pci": {Value: "nomis", Operation: "delete"}, // delete kv + "debugpat": {Value: "", Operation: "add"}, // passed key, operation is invalid, default to add key + "audit": {Value: "1", Operation: "add"}, // passed kv, key is inexistent, operation is invalid, default to add kv + "nomodeset": {Value: "1", Operation: "delete"}, // delete key with inconsistent value + }, + }, + }, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro=1\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debugpat\saudit=1|audit=1\sdebugpat)$`, wantErr: false, }, { @@ -229,31 +302,35 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri args: args{ config: &agent.SysConfig{ Contents: map[string]*agent.KeyInfo{ - "panic": {Value: "4"}, - "quiet": {Value: "", Operation: "delete"}, - "oops": {Value: ""}, // update existent kv with null value - "selinux": {Value: "1", Operation: "delete"}, - "acpi": {Value: "off", Operation: "delete"}, - "debug": {}, - "pci": {Value: "nomis"}, + "debug": {}, + "pci": {Value: "nomis"}, + "quiet": {Value: "", Operation: "delete"}, + "panic": {Value: "4"}, + "nomodeset": {Operation: "update"}, // invalid operation, default to update existent key + "softlockup_panic": {Value: "0", Operation: "update"}, // invalid operation, default to update existent kv + "oops": {Value: ""}, // update existent kv with null value + "": {Value: "test"}, // warning, skip, failed to add kv with empty key + "selinux": {Value: "1", Operation: "delete"}, + "acpi": {Value: "off", Operation: "delete"}, + "ro": {Value: ""}, }, }, }, - pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=1\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=4\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=4\s+(debug\spci=nomis|pci=nomis\sdebug)$`, wantErr: false, }, } patchGetGrubPath := gomonkey.ApplyFuncReturn(getGrubCfgPath, grubCfgPath) defer patchGetGrubPath.Reset() patchGetConfigPartition := gomonkey.ApplyFuncSeq(getConfigPartition, []gomonkey.OutputCell{ + {Values: gomonkey.Params{false, nil}}, {Values: gomonkey.Params{false, nil}}, {Values: gomonkey.Params{true, nil}}, }) defer patchGetConfigPartition.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - g := GrubCmdline{} - if err := g.SetConfig(tt.args.config); (err != nil) != tt.wantErr { + if err := tt.g.SetConfig(tt.args.config); (err != nil) != tt.wantErr { t.Errorf("GrubCmdline.SetConfig() error = %v, wantErr %v", err, tt.wantErr) } contents, err := os.ReadFile(grubCfgPath) @@ -375,3 +452,101 @@ func Test_getGrubCfgPath(t *testing.T) { }) } } + +func Test_getConfigPartition(t *testing.T) { + type args struct { + isCurPartition bool + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "get current partition", + args: args{isCurPartition: true}, + want: false, + wantErr: false, + }, + { + name: "get next partition", + args: args{isCurPartition: false}, + want: true, + wantErr: false, + }, + } + patchRootfsDisks := gomonkey.ApplyFuncReturn(getRootfsDisks, "/dev/sda2", "/dev/sda3", nil) + defer patchRootfsDisks.Reset() + // assume now is partition A, want to swiching to partition B + patchGetNextPartition := gomonkey.ApplyFuncReturn(getNextPart, "/dev/sda3", "B", nil) + defer patchGetNextPartition.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getConfigPartition(tt.args.isCurPartition) + if (err != nil) != tt.wantErr { + t.Errorf("getConfigPartition() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getConfigPartition() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_ConfigFactoryTemplate(t *testing.T) { + type args struct { + configType string + config *agent.SysConfig + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "error", + args: args{ + configType: "test", + config: &agent.SysConfig{}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := ConfigFactoryTemplate(tt.args.configType, tt.args.config); (err != nil) != tt.wantErr { + t.Errorf("ConfigFactoryTemplate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_convertNewConfigsToString(t *testing.T) { + type args struct { + newConfigs []string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "error", args: args{newConfigs: []string{"a"}}, want: "", wantErr: true}, + } + patchFprintf := gomonkey.ApplyFuncReturn(fmt.Fprintf, 0, fmt.Errorf("error")) + defer patchFprintf.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := convertNewConfigsToString(tt.args.newConfigs) + if (err != nil) != tt.wantErr { + t.Errorf("convertNewConfigsToString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("convertNewConfigsToString() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/agent/server/containerd_image.go b/cmd/agent/server/containerd_image.go index f180fb54a74ea7417daaa7c85072231ca3d570a0..fd612745252b8c73e3482500c3b454e321ab5b74 100644 --- a/cmd/agent/server/containerd_image.go +++ b/cmd/agent/server/containerd_image.go @@ -23,9 +23,7 @@ import ( pb "openeuler.org/KubeOS/cmd/agent/api" ) -var ( - defaultNamespace = "k8s.io" -) +const defaultNamespace = "k8s.io" type conImageHandler struct{} @@ -56,7 +54,7 @@ func (c conImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath prep } } else { containerdCommand = "ctr" - if err := runCommand("ctr", "-n", defaultNamespace, "images", "pull", "--host-dir", + if err := runCommand("ctr", "-n", defaultNamespace, "images", "pull", "--hosts-dir", "/etc/containerd/certs.d", imageName); err != nil { return "", err } @@ -76,7 +74,7 @@ func (c conImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath prep return "", err } defer checkAndCleanMount(mountPath) - if err := copyFile(neededPath.tarPath, mountPath+"/"+rootfsArchive); err != nil { + if err := copyFile(neededPath.tarPath, mountPath+"/"+neededPath.rootfsFile); err != nil { return "", err } return "", nil diff --git a/cmd/agent/server/containerd_image_test.go b/cmd/agent/server/containerd_image_test.go new file mode 100644 index 0000000000000000000000000000000000000000..85347c83296a28a42131639edd4f7895014a52e1 --- /dev/null +++ b/cmd/agent/server/containerd_image_test.go @@ -0,0 +1,147 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package server implements server of os-agent and listener of os-agent server. The server uses gRPC interface. +package server + +import ( + "os" + "testing" + + "github.com/agiledragon/gomonkey/v2" + pb "openeuler.org/KubeOS/cmd/agent/api" +) + +func Test_conImageHandler_downloadImage(t *testing.T) { + type args struct { + req *pb.UpdateRequest + } + tests := []struct { + name string + c conImageHandler + args args + want string + wantErr bool + }{ + { + name: "pullImageError", + c: conImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "testError"}, + }, + want: "", + wantErr: true, + }, + { + name: "checkSumError", + c: conImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "docker.io/library/hello-world:latest"}, + }, + want: "", + wantErr: true, + }, + { + name: "normal", + c: conImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ + ContainerImage: "docker.io/library/hello-world:latest", + }, + }, + want: "update-test1/upadte.img", + wantErr: false, + }, + { + name: "invalid image name", + c: conImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "nginx;v1"}, + }, + want: "", + wantErr: true, + }, + } + patchPrepareEnv := gomonkey.ApplyFunc(prepareEnv, func() (preparePath, error) { + return preparePath{updatePath: "update-test1/", + mountPath: "update-test1/mountPath", + tarPath: "update-test1/mountPath/hello", + imagePath: "update-test1/upadte.img", + rootfsFile: "hello"}, nil + }) + defer patchPrepareEnv.Reset() + patchCreateOSImage := gomonkey.ApplyFunc(createOSImage, func(neededPath preparePath) (string, error) { + return "update-test1/upadte.img", nil + }) + defer patchCreateOSImage.Reset() + + if err := os.MkdirAll("update-test1/mountPath", os.ModePerm); err != nil { + t.Errorf("create test dir error = %v", err) + return + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := conImageHandler{} + if tt.name == "normal" { + imageDigests, err := getOCIImageDigest("crictl", "docker.io/library/hello-world:latest") + if err != nil { + t.Errorf("conImageHandler.getRootfsArchive() get oci image digests error = %v", err) + } + tt.args.req.CheckSum = imageDigests + } + got, err := c.downloadImage(tt.args.req) + if (err != nil) != tt.wantErr { + t.Errorf("conImageHandler.downloadImage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("conImageHandler.downloadImage() = %v, want %v", got, tt.want) + } + }) + } + defer func() { + if err := runCommand("crictl", "rmi", "docker.io/library/hello-world:latest"); err != nil { + t.Errorf("remove kubeos-temp container error = %v", err) + } + if err := os.RemoveAll("update-test1"); err != nil { + t.Errorf("remove update-test error = %v", err) + } + }() +} + +func Test_copyFile(t *testing.T) { + type args struct { + dstFileName string + srcFileName string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "srcFileNotExist", + args: args{ + dstFileName: "bbb.txt", + srcFileName: "aaa.txt", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := copyFile(tt.args.dstFileName, tt.args.srcFileName); (err != nil) != tt.wantErr { + t.Errorf("copyFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/agent/server/disk_image.go b/cmd/agent/server/disk_image.go index c89003b9f530e8d148fa4e918ca2fc0fc8a7ee14..8bd6bf6755bf2c6a39b0cece17893a977c2dc75c 100644 --- a/cmd/agent/server/disk_image.go +++ b/cmd/agent/server/disk_image.go @@ -153,7 +153,7 @@ func loadCaCerts(caCert string) (*http.Client, error) { if err != nil { return &http.Client{}, err } - ca, err := ioutil.ReadFile(certPath + caCert) + ca, err := ioutil.ReadFile(getCertPath() + caCert) if err != nil { return &http.Client{}, fmt.Errorf("read the ca certificate error %s", err) } @@ -173,7 +173,7 @@ func loadClientCerts(caCert, clientCert, clientKey string) (*http.Client, error) if err != nil { return &http.Client{}, err } - ca, err := ioutil.ReadFile(certPath + caCert) + ca, err := ioutil.ReadFile(getCertPath() + caCert) if err != nil { return &http.Client{}, err } @@ -186,7 +186,7 @@ func loadClientCerts(caCert, clientCert, clientKey string) (*http.Client, error) if err != nil { return &http.Client{}, err } - cliCrt, err := tls.LoadX509KeyPair(certPath+clientCert, certPath+clientKey) + cliCrt, err := tls.LoadX509KeyPair(getCertPath()+clientCert, getCertPath()+clientKey) if err != nil { return &http.Client{}, err } @@ -206,7 +206,7 @@ func certExist(certFile string) error { if certFile == "" { return fmt.Errorf("please provide the certificate") } - _, err := os.Stat(certPath + certFile) + _, err := os.Stat(getCertPath() + certFile) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("certificate is not exist %s ", err) diff --git a/cmd/agent/server/disk_image_test.go b/cmd/agent/server/disk_image_test.go index 3c821131f3f7a96b35aa7e11d412cc753d584276..f970bd7d07b4cfda8a3b0e875cdf17d3248b79e0 100644 --- a/cmd/agent/server/disk_image_test.go +++ b/cmd/agent/server/disk_image_test.go @@ -14,19 +14,37 @@ package server import ( - "crypto/tls" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "fmt" + "io" + "io/fs" + "math/big" "net/http" "os" "reflect" + "strings" + "syscall" "testing" + "time" "github.com/agiledragon/gomonkey/v2" - pb "openeuler.org/KubeOS/cmd/agent/api" ) -func Testdownload(t *testing.T) { +func Test_download(t *testing.T) { + tmpDir := t.TempDir() + tmpFileForDownload := tmpDir + "/tmpFileForDownload" + tmpFile, err := os.Create(tmpFileForDownload) + if err != nil { + t.Errorf("open file error: %v", err) + } + defer tmpFile.Close() type args struct { req *pb.UpdateRequest } @@ -37,14 +55,77 @@ func Testdownload(t *testing.T) { wantErr bool }{ {name: "errornil", args: args{&pb.UpdateRequest{Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, - {name: "normal", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "/persist/update.img", wantErr: false}, - {name: "errornodir", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + {name: "error response", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.abc", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + { + name: "normal", + args: args{ + req: &pb.UpdateRequest{ + ImageUrl: "http://www.openeuler.org/zh/", + FlagSafe: true, + Certs: &pb.CertsInfo{}, + }, + }, + want: tmpFileForDownload, + wantErr: false, + }, + { + name: "disk space not enough", + args: args{ + req: &pb.UpdateRequest{ + ImageUrl: "http://www.openeuler.org/zh/", + FlagSafe: true, + Certs: &pb.CertsInfo{}, + }, + }, + want: "", + wantErr: true, + }, } + var patchStatfs *gomonkey.Patches + patchStatfs = gomonkey.ApplyFunc(syscall.Statfs, func(path string, stat *syscall.Statfs_t) error { + stat.Bfree = 3000 + stat.Bsize = 4096 + return nil + }) + defer patchStatfs.Reset() + patchGetImageUrl := gomonkey.ApplyFuncSeq(getImageURL, + []gomonkey.OutputCell{ + {Values: gomonkey.Params{&http.Response{}, fmt.Errorf("error")}}, + {Values: gomonkey.Params{&http.Response{StatusCode: http.StatusBadRequest, Body: io.NopCloser(strings.NewReader(""))}, nil}}, + { + Values: gomonkey.Params{ + &http.Response{ + StatusCode: http.StatusOK, + ContentLength: 5, + Body: io.NopCloser(strings.NewReader("hello")), + }, + nil, + }, + }, + { + Values: gomonkey.Params{ + &http.Response{ + StatusCode: http.StatusOK, + ContentLength: 5, + Body: io.NopCloser(strings.NewReader("hello")), + }, + nil, + }, + }, + }, + ) + defer patchGetImageUrl.Reset() + patchOSCreate := gomonkey.ApplyFuncReturn(os.Create, tmpFile, nil) + defer patchOSCreate.Reset() for _, tt := range tests { - if tt.name == "normal" { - os.Mkdir("/persist", os.ModePerm) - } t.Run(tt.name, func(t *testing.T) { + if tt.name == "disk space not enough" { + patchStatfs = gomonkey.ApplyFunc(syscall.Statfs, func(path string, stat *syscall.Statfs_t) error { + stat.Bfree = 1 + stat.Bsize = 4096 + return nil + }) + } got, err := download(tt.args.req) if (err != nil) != tt.wantErr { t.Errorf("download() error = %v, wantErr %v", err, tt.wantErr) @@ -54,47 +135,43 @@ func Testdownload(t *testing.T) { t.Errorf("download() got = %v, want %v", got, tt.want) } }) - if tt.name == "normal" { - os.RemoveAll("/persist") - } } } -func TestcheckSumMatch(t *testing.T) { +func Test_checkSumMatch(t *testing.T) { + tmpDir := t.TempDir() + tmpFileForCheckSum := tmpDir + "/tmpFileForCheckSum" + err := os.WriteFile(tmpFileForCheckSum, []byte("hello"), 0644) + if err != nil { + t.Errorf("open file error: %v", err) + } type args struct { filePath string checkSum string } - ff, _ := os.Create("aa.txt") - ff.Chmod(os.ModePerm) tests := []struct { name string args args wantErr bool }{ - {name: "error", args: args{filePath: "aaa", checkSum: "aaa"}, wantErr: true}, - {name: "errordir", args: args{filePath: "/aaa", checkSum: "/aaa"}, wantErr: true}, - {name: "errortxt", args: args{filePath: "aa.txt", checkSum: "aa.txt"}, wantErr: true}, + { + name: "normal", + args: args{filePath: tmpFileForCheckSum, checkSum: calculateChecksum("hello")}, + wantErr: false, + }, + {name: "error", args: args{filePath: tmpFileForCheckSum, checkSum: "aaa"}, wantErr: true}, + {name: "unfound error", args: args{filePath: "", checkSum: "aaa"}, wantErr: true}, } for _, tt := range tests { - if tt.name == "errordir" { - os.Mkdir("/aaa", os.ModePerm) - } t.Run(tt.name, func(t *testing.T) { if err := checkSumMatch(tt.args.filePath, tt.args.checkSum); (err != nil) != tt.wantErr { t.Errorf("checkSumMatch() error = %v, wantErr %v", err, tt.wantErr) } }) - if tt.name == "errordir" { - os.RemoveAll("/aaa") - } } - defer os.Remove("aa.txt") - defer ff.Close() - } -func TestgetImageURL(t *testing.T) { +func Test_getImageURL(t *testing.T) { type args struct { req *pb.UpdateRequest } @@ -105,34 +182,66 @@ func TestgetImageURL(t *testing.T) { wantErr bool }{ {name: "httpNotSafe", args: args{req: &pb.UpdateRequest{ - ImageUrl: "http://www.openeuler.org/zh/", + ImageUrl: "http://www.openeuler.abc/zh/", FlagSafe: false, MTLS: false, Certs: &pb.CertsInfo{}, }}, want: &http.Response{}, wantErr: true}, - {name: "mTLSError", args: args{req: &pb.UpdateRequest{ - ImageUrl: "http://www.openeuler.org/zh/", + {name: "httpSuccess", args: args{req: &pb.UpdateRequest{ + ImageUrl: "http://www.openeuler.abc/zh/", + FlagSafe: true, + MTLS: false, + Certs: &pb.CertsInfo{}, + }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, + {name: "mTLSGetSuccess", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", FlagSafe: true, MTLS: true, Certs: &pb.CertsInfo{}, - }}, want: &http.Response{}, wantErr: true}, - {name: "httpsError", args: args{req: &pb.UpdateRequest{ - ImageUrl: "https://www.openeuler.org/zh/", + }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, + {name: "httpsGetSuccess", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", FlagSafe: true, MTLS: false, Certs: &pb.CertsInfo{}, + }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, + {name: "httpsLoadCertsError", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", + FlagSafe: true, + MTLS: false, + Certs: &pb.CertsInfo{}, + }}, want: &http.Response{}, wantErr: true}, + {name: "httpsMLTSLoadCertsError", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", + FlagSafe: true, + MTLS: true, + Certs: &pb.CertsInfo{}, }}, want: &http.Response{}, wantErr: true}, } - patchLoadClientCerts := gomonkey.ApplyFunc(loadClientCerts, func(caCert, clientCert, clientKey string) (*http.Client, error) { - return &http.Client{}, nil + patchLoadClientCerts := gomonkey.ApplyFuncSeq(loadClientCerts, []gomonkey.OutputCell{ + {Values: gomonkey.Params{&http.Client{}, nil}}, + {Values: gomonkey.Params{&http.Client{}, fmt.Errorf("error")}}, }) defer patchLoadClientCerts.Reset() - patchLoadCaCerts := gomonkey.ApplyFunc(loadCaCerts, func(caCert string) (*http.Client, error) { - return &http.Client{}, nil + patchLoadCaCerts := gomonkey.ApplyFuncSeq(loadCaCerts, []gomonkey.OutputCell{ + {Values: gomonkey.Params{&http.Client{}, nil}}, + {Values: gomonkey.Params{&http.Client{}, fmt.Errorf("error")}}, }) defer patchLoadCaCerts.Reset() + patchGet := gomonkey.ApplyFunc(http.Get, func(url string) (resp *http.Response, err error) { + return &http.Response{StatusCode: http.StatusOK}, nil + }) + defer patchGet.Reset() + patchClientGet := gomonkey.ApplyMethod(reflect.TypeOf(&http.Client{}), "Get", func(_ *http.Client, url string) (resp *http.Response, err error) { + return &http.Response{StatusCode: http.StatusOK}, nil + }) + defer patchClientGet.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.name == "httpSuccess" { + patchGet := gomonkey.ApplyFuncReturn(http.Get, &http.Response{StatusCode: http.StatusOK}, nil) + defer patchGet.Reset() + } got, err := getImageURL(tt.args.req) if (err != nil) != tt.wantErr { t.Errorf("getImageURL() error = %v, wantErr %v", err, tt.wantErr) @@ -145,20 +254,29 @@ func TestgetImageURL(t *testing.T) { } } -func TestloadCaCerts(t *testing.T) { +func Test_loadCaCerts(t *testing.T) { + tmpDir := t.TempDir() + caPath := tmpDir + "/fake.crt" + createFakeCertKey(caPath, "") type args struct { caCert string } tests := []struct { name string args args - want *http.Client wantErr bool }{ - {name: "noCaCertError", args: args{caCert: "bb.txt"}, want: &http.Client{}, wantErr: true}, + { + name: "normal", + args: args{ + caCert: caPath, + }, + wantErr: false, + }, + {name: "no cert", args: args{caCert: ""}, wantErr: true}, } - os.MkdirAll(certPath, 0644) - defer os.RemoveAll(certPath) + patchGetCertPath := gomonkey.ApplyFuncReturn(getCertPath, "") + defer patchGetCertPath.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := loadCaCerts(tt.args.caCert) @@ -166,48 +284,39 @@ func TestloadCaCerts(t *testing.T) { t.Errorf("loadCaCerts() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("loadCaCerts() = %v, want %v", got, tt.want) + if got == nil { + t.Errorf("loadCaCerts() = %v", got) } }) } } -func TestloadClientCerts(t *testing.T) { +func Test_loadClientCerts(t *testing.T) { + tmpDir := t.TempDir() + clientCertPath := tmpDir + "/fakeClientCert.crt" + clientKeyPath := tmpDir + "/fakeClientKey.crt" + createFakeCertKey(clientCertPath, clientKeyPath) type args struct { caCert string clientCert string clientKey string } - pool := &x509.CertPool{} tests := []struct { name string args args - want *http.Client wantErr bool }{ - {name: "noCaCertError", args: args{" dd.txt", "bb.txt", "cc.txt"}, want: &http.Client{}, wantErr: true}, - {name: "noClientCertError", args: args{"ca.crt", "bb.txt", "cc.txt"}, want: &http.Client{}, wantErr: true}, - {name: "noClientKeyError", args: args{"ca.crt", "client.crt", "cc.txt"}, want: &http.Client{}, wantErr: true}, - } - os.MkdirAll(certPath, 0644) - caFile, _ := os.Create(certPath + "ca.crt") - clientCertFile, _ := os.Create(certPath + "client.crt") - clientKeyFile, _ := os.Create(certPath + "client.key") - - patchNewCertPool := gomonkey.ApplyFunc(x509.NewCertPool, func() *x509.CertPool { - return pool - }) - defer patchNewCertPool.Reset() - patchAppendCertsFromPEM := gomonkey.ApplyMethod(reflect.TypeOf(pool), "AppendCertsFromPEM", func(_ *x509.CertPool, _ []byte) (ok bool) { - return true - }) - defer patchAppendCertsFromPEM.Reset() - patchLoadX509KeyPair := gomonkey.ApplyFunc(tls.LoadX509KeyPair, func(certFile string, keyFile string) (tls.Certificate, error) { - return tls.Certificate{}, nil - }) - defer patchLoadX509KeyPair.Reset() + { + name: "normal", + args: args{ + caCert: clientCertPath, clientCert: clientCertPath, clientKey: clientKeyPath, + }, + wantErr: false, + }, + } + patchGetCertPath := gomonkey.ApplyFuncReturn(getCertPath, "") + defer patchGetCertPath.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := loadClientCerts(tt.args.caCert, tt.args.clientCert, tt.args.clientKey) @@ -215,19 +324,14 @@ func TestloadClientCerts(t *testing.T) { t.Errorf("loadClientCerts() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("loadClientCerts() got = %v, want %v", got, tt.want) + if got == nil { + t.Errorf("loadClientCerts() got = %v", got) } }) } - caFile.Close() - clientCertFile.Close() - clientKeyFile.Close() - defer os.RemoveAll("/etc/KubeOS") - } -func TestcertExist(t *testing.T) { +func Test_certExist(t *testing.T) { type args struct { certFile string } @@ -238,17 +342,130 @@ func TestcertExist(t *testing.T) { }{ {name: "fileEmpty", args: args{certFile: ""}, wantErr: true}, {name: "fileNotExist", args: args{certFile: "bb.txt"}, wantErr: true}, - {name: "normal", args: args{certFile: "aa.txt"}, wantErr: false}, + {name: "unknow error", args: args{certFile: "cc.txt"}, wantErr: true}, } - os.MkdirAll(certPath, 0644) - ff, _ := os.Create(certPath + "aa.txt") for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + var patchStat *gomonkey.Patches + if tt.name == "unknow error" { + patchStat = gomonkey.ApplyFunc(os.Stat, func(name string) (fs.FileInfo, error) { + return fs.FileInfo(nil), fmt.Errorf("error") + }) + } if err := certExist(tt.args.certFile); (err != nil) != tt.wantErr { t.Errorf("certExist() error = %v, wantErr %v", err, tt.wantErr) } + if tt.name == "unknow error" { + patchStat.Reset() + } }) } - ff.Close() defer os.RemoveAll("/etc/KubeOS/") } + +func createFakeCertKey(certPath, keyPath string) { + privateKey, _ := rsa.GenerateKey(rand.Reader, 2048) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "Fake Client Certificate", + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + certBytes, _ := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certBytes}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}) + os.WriteFile(certPath, certPEM, 0644) + if keyPath != "" { + os.WriteFile(keyPath, keyPEM, 0644) + } +} + +func calculateChecksum(data string) string { + hash := sha256.New() + hash.Write([]byte(data)) + return hex.EncodeToString(hash.Sum(nil)) +} + +func Test_diskHandler_getRootfsArchive(t *testing.T) { + type args struct { + req *pb.UpdateRequest + neededPath preparePath + } + tests := []struct { + name string + d diskHandler + args args + want string + wantErr bool + }{ + { + name: "normal", d: diskHandler{}, + args: args{req: &pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/"}, neededPath: preparePath{}}, + want: "/persist/update.img", + wantErr: false, + }, + { + name: "error", d: diskHandler{}, + args: args{req: &pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/"}, neededPath: preparePath{}}, + want: "", + wantErr: true, + }, + } + patchDownload := gomonkey.ApplyFuncSeq(download, []gomonkey.OutputCell{ + {Values: gomonkey.Params{"/persist/update.img", nil}}, + {Values: gomonkey.Params{"", fmt.Errorf("error")}}, + }) + defer patchDownload.Reset() + patchCheckSumMatch := gomonkey.ApplyFuncReturn(checkSumMatch, nil) + defer patchCheckSumMatch.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := diskHandler{} + got, err := d.getRootfsArchive(tt.args.req, tt.args.neededPath) + if (err != nil) != tt.wantErr { + t.Errorf("diskHandler.getRootfsArchive() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("diskHandler.getRootfsArchive() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_diskHandler_downloadImage(t *testing.T) { + type args struct { + req *pb.UpdateRequest + } + tests := []struct { + name string + d diskHandler + args args + want string + wantErr bool + }{ + {name: "normal", d: diskHandler{}, args: args{req: &pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/"}}, want: "/persist/update.img", wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := diskHandler{} + patchGetRootfsArchive := gomonkey.ApplyPrivateMethod(reflect.TypeOf(d), "getRootfsArchive", func(_ *diskHandler, _ *pb.UpdateRequest, _ preparePath) (string, error) { + return "/persist/update.img", nil + }) + got, err := d.downloadImage(tt.args.req) + if (err != nil) != tt.wantErr { + t.Errorf("diskHandler.downloadImage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("diskHandler.downloadImage() = %v, want %v", got, tt.want) + } + patchGetRootfsArchive.Reset() + }) + } +} diff --git a/cmd/agent/server/docker_image.go b/cmd/agent/server/docker_image.go index 23e596b3beec90b2eb0882529023e77086489946..16bcea5467786febca37eebd35615ab7cfe3b3e4 100644 --- a/cmd/agent/server/docker_image.go +++ b/cmd/agent/server/docker_image.go @@ -61,13 +61,13 @@ func (d dockerImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath p if err != nil { return "", err } - if err := runCommand("docker", "cp", containerId+":/"+rootfsArchive, neededPath.updatePath); err != nil { - return "", err - } defer func() { if err := runCommand("docker", "rm", containerId); err != nil { logrus.Errorln("remove kubeos-temp container error", err) } }() + if err := runCommand("docker", "cp", containerId+":/"+neededPath.rootfsFile, neededPath.updatePath); err != nil { + return "", err + } return neededPath.tarPath, nil } diff --git a/cmd/agent/server/docker_image_test.go b/cmd/agent/server/docker_image_test.go index 9987939390dbe33dbc3a97d026352d0fc4002d24..2dbf33700f7cfb499d2a00dfaa89bc48cf6300b8 100644 --- a/cmd/agent/server/docker_image_test.go +++ b/cmd/agent/server/docker_image_test.go @@ -17,38 +17,102 @@ import ( "os" "testing" + "github.com/agiledragon/gomonkey/v2" pb "openeuler.org/KubeOS/cmd/agent/api" ) -func TestpullOSImage(t *testing.T) { +func Test_dockerImageHandler_downloadImage(t *testing.T) { type args struct { req *pb.UpdateRequest } - os.Mkdir("/persist", os.ModePerm) tests := []struct { name string + d dockerImageHandler args args want string wantErr bool }{ - {name: "pull image error", args: args{req: &pb.UpdateRequest{ - DockerImage: "test", - }}, want: "", wantErr: true}, - {name: "normal", args: args{req: &pb.UpdateRequest{ - DockerImage: "centos", - }}, want: "/persist/update.img", wantErr: false}, + { + name: "pullImageError", + d: dockerImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "testError"}, + }, + want: "", + wantErr: true, + }, + + { + name: "checkSumError", + d: dockerImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "hello-world", CheckSum: "aaaaaa"}, + }, + want: "", + wantErr: true, + }, + + { + name: "normal", + d: dockerImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "hello-world"}, + }, + want: "update-test/upadte.img", + wantErr: false, + }, + } + patchPrepareEnv := gomonkey.ApplyFunc(prepareEnv, func() (preparePath, error) { + return preparePath{updatePath: "update-test/", + mountPath: "update-test/mountPath", + tarPath: "update-test/mountPath/hello", + imagePath: "update-test/upadte.img", + rootfsFile: "hello"}, nil + }) + defer patchPrepareEnv.Reset() + + patchCreateOSImage := gomonkey.ApplyFunc(createOSImage, func(neededPath preparePath) (string, error) { + return "update-test/upadte.img", nil + }) + defer patchCreateOSImage.Reset() + + if err := os.MkdirAll("update-test/mountPath", os.ModePerm); err != nil { + t.Errorf("create test dir error = %v", err) + return } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := pullOSImage(tt.args.req) + if tt.name == "normal" { + _, err := runCommandWithOut("docker", "create", "--name", "kubeos-temp", "hello-world") + if err != nil { + t.Errorf("Test_dockerImageHandler_getRootfsArchive create container error = %v", err) + return + } + imageDigests, err := getOCIImageDigest("docker", "hello-world") + + if err != nil { + t.Errorf("Test_dockerImageHandler_getRootfsArchive get oci image digests error = %v", err) + } + tt.args.req.CheckSum = imageDigests + } + d := dockerImageHandler{} + got, err := d.downloadImage(tt.args.req) if (err != nil) != tt.wantErr { - t.Errorf("pullOSImage() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("dockerImageHandler.downloadImage() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("pullOSImage() = %v, want %v", got, tt.want) + t.Errorf("dockerImageHandler.downloadImage() = %v, want %v", got, tt.want) } }) } - defer os.RemoveAll("/persist") + defer func() { + if err := runCommand("docker", "rmi", "hello-world"); err != nil { + t.Errorf("remove kubeos-temp container error = %v", err) + } + if err := os.RemoveAll("update-test"); err != nil { + t.Errorf("remove update-test error = %v", err) + } + }() } diff --git a/cmd/agent/server/server.go b/cmd/agent/server/server.go index b41ebc497ef361d19eaa89b12541914d1655c2d8..f8cbb41063475cdfc4b58c97719c68a6e025135e 100644 --- a/cmd/agent/server/server.go +++ b/cmd/agent/server/server.go @@ -112,6 +112,15 @@ func (s *Server) update(req *pb.UpdateRequest) error { return fmt.Errorf("image type %s cannot be recognized", action) } imagePath, err := handler.downloadImage(req) + defer func() { + if err != nil { + path := newPreparePath() + if err := cleanSpace(path.updatePath, path.mountPath, path.imagePath); err != nil { + logrus.Errorln("clean space error " + err.Error()) + } + logrus.Infoln("clean space success") + } + }() if err != nil { return err } @@ -171,3 +180,7 @@ func (s *Server) reboot() error { } return syscall.Reboot(syscall.LINUX_REBOOT_CMD_RESTART) } + +func getCertPath() string { + return certPath +} diff --git a/cmd/agent/server/server_test.go b/cmd/agent/server/server_test.go index 0aac36ab816bed38c7ee0c66d464013f7445b9de..15b6f5ea333af1da1d43b333d7115dbc1fda4a9c 100644 --- a/cmd/agent/server/server_test.go +++ b/cmd/agent/server/server_test.go @@ -89,7 +89,21 @@ func TestServerUpdate(t *testing.T) { {name: "error", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, args: args{in0: context.Background(), req: &pb.UpdateRequest{Version: "test", Certs: &pb.CertsInfo{}}}, want: &pb.UpdateResponse{}, wantErr: true}, + {name: "success", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, + args: args{in0: context.Background(), req: &pb.UpdateRequest{Version: "test", Certs: &pb.CertsInfo{}, ImageType: "containerd"}}, + want: &pb.UpdateResponse{}, wantErr: false}, } + patchRootfsDisks := gomonkey.ApplyFuncReturn(getRootfsDisks, "/dev/sda2", "/dev/sda3", nil) + defer patchRootfsDisks.Reset() + // assume now is partition A, want to swiching to partition B + patchGetNextPartition := gomonkey.ApplyFuncReturn(getNextPart, "/dev/sda3", "B", nil) + defer patchGetNextPartition.Reset() + patchDownloadImage := gomonkey.ApplyPrivateMethod(conImageHandler{}, "downloadImage", func(_ conImageHandler, req *pb.UpdateRequest) (string, error) { + return "", nil + }) + defer patchDownloadImage.Reset() + patchInstall := gomonkey.ApplyFuncReturn(install, nil) + defer patchInstall.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &Server{ @@ -129,11 +143,26 @@ func TestServerRollback(t *testing.T) { {name: "error", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, args: args{in0: context.Background(), req: &pb.RollbackRequest{}}, want: &pb.RollbackResponse{}, wantErr: true}, + {name: "success", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, + args: args{in0: context.Background(), req: &pb.RollbackRequest{}}, + want: &pb.RollbackResponse{}, wantErr: false}, } - patchGetNextPart := gomonkey.ApplyFunc(getNextPart, func(partA string, partB string) (string, string, error) { - return "", "", fmt.Errorf("rollbak test error") + patchRootfsDisks := gomonkey.ApplyFuncReturn(getRootfsDisks, "/dev/sda2", "/dev/sda3", nil) + defer patchRootfsDisks.Reset() + // assume now is partition A, want to swiching to partition B + patchGetNextPartition := gomonkey.ApplyFuncSeq(getNextPart, []gomonkey.OutputCell{ + {Values: gomonkey.Params{"", "", fmt.Errorf("rollbak test error")}}, + {Values: gomonkey.Params{"/dev/sda3", "B", nil}}, }) - defer patchGetNextPart.Reset() + defer patchGetNextPartition.Reset() + patchDownloadImage := gomonkey.ApplyPrivateMethod(conImageHandler{}, "downloadImage", func(_ conImageHandler, req *pb.UpdateRequest) (string, error) { + return "", nil + }) + defer patchDownloadImage.Reset() + patchInstall := gomonkey.ApplyFuncReturn(install, nil) + defer patchInstall.Reset() + patchRunCommand := gomonkey.ApplyFuncReturn(runCommand, nil) + defer patchRunCommand.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &Server{ @@ -179,9 +208,9 @@ func TestServerupdate(t *testing.T) { }}, wantErr: true}, {name: "errordocker", args: args{&pb.UpdateRequest{ - DockerImage: "", - ImageType: "docker", - Certs: &pb.CertsInfo{}, + ContainerImage: "", + ImageType: "docker", + Certs: &pb.CertsInfo{}, }}, wantErr: true}, } @@ -264,3 +293,47 @@ func TestServerreboot(t *testing.T) { }) } } + +func TestServer_Configure(t *testing.T) { + type fields struct { + UnimplementedOSServer pb.UnimplementedOSServer + mutex Lock + disableReboot bool + } + type args struct { + in0 context.Context + req *pb.ConfigureRequest + } + tests := []struct { + name string + fields fields + args args + want *pb.ConfigureResponse + wantErr bool + }{ + { + name: "nil", + fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, + args: args{in0: context.Background(), req: &pb.ConfigureRequest{}}, + want: &pb.ConfigureResponse{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Server{ + UnimplementedOSServer: tt.fields.UnimplementedOSServer, + mutex: tt.fields.mutex, + disableReboot: tt.fields.disableReboot, + } + got, err := s.Configure(tt.args.in0, tt.args.req) + if (err != nil) != tt.wantErr { + t.Errorf("Server.Configure() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Server.Configure() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/agent/server/utils.go b/cmd/agent/server/utils.go index c8a72c3e579626f6f95e869e47a3241bb5b79bb3..fdddc7d5d5af1303c1c8a46a42d8398f4e289de0 100644 --- a/cmd/agent/server/utils.go +++ b/cmd/agent/server/utils.go @@ -31,10 +31,7 @@ import ( const ( needGBSize = 3 // the max size of update files needed // KB is 1024 B - KB = 1024 -) - -var ( + KB = 1024 rootfsArchive = "os.tar" updateDir = "KubeOS-Update" mountDir = "kubeos-update" @@ -51,6 +48,7 @@ type preparePath struct { mountPath string tarPath string imagePath string + rootfsFile string } func runCommand(name string, args ...string) error { @@ -77,6 +75,9 @@ func deleteNewline(out string) string { } func install(imagePath string, side string, next string) error { + if err := modifyImageLabel(imagePath, side, next); err != nil { + return err + } if err := runCommand("dd", "if="+imagePath, "of="+side, "bs=8M"); err != nil { return err } @@ -100,12 +101,10 @@ func getNextPart(partA string, partB string) (string, string, error) { mountPoint := strings.TrimSpace(string(out)) side := partA + next := "A" if mountPoint == "/" { side = partB - } - next := "B" - if side != partB { - next = "A" + next = "B" } return side, next, nil } @@ -192,24 +191,25 @@ func prepareEnv() (preparePath, error) { if err := checkDiskSize(needGBSize, PersistDir); err != nil { return preparePath{}, err } - updatePath := splicePath(PersistDir, updateDir) - mountPath := splicePath(updatePath, mountDir) - tarPath := splicePath(updatePath, rootfsArchive) - imagePath := splicePath(PersistDir, osImageName) - - if err := cleanSpace(updatePath, mountPath, imagePath); err != nil { + upgradePath := newPreparePath() + if err := cleanSpace(upgradePath.updatePath, upgradePath.mountPath, upgradePath.imagePath); err != nil { return preparePath{}, err } - if err := os.MkdirAll(mountPath, imgPermission); err != nil { + if err := os.MkdirAll(upgradePath.mountPath, imgPermission); err != nil { return preparePath{}, err } - upgradePath := preparePath{ + return upgradePath, nil +} + +func newPreparePath() preparePath { + updatePath := splicePath(PersistDir, updateDir) + return preparePath{ updatePath: updatePath, - mountPath: mountPath, - tarPath: tarPath, - imagePath: imagePath, + mountPath: splicePath(updatePath, mountDir), + tarPath: splicePath(updatePath, rootfsArchive), + imagePath: splicePath(PersistDir, osImageName), + rootfsFile: rootfsArchive, } - return upgradePath, nil } func checkDiskSize(needGBSize int, path string) error { @@ -284,41 +284,81 @@ func checkFileExist(path string) (bool, error) { } func checkOCIImageDigestMatch(containerRuntime string, imageName string, checkSum string) error { + imageDigests, err := getOCIImageDigest(containerRuntime, imageName) + if err != nil { + return err + } + if imageDigests == "" { + logrus.Errorln("error when get ", imageName, " digests") + return fmt.Errorf("error when get %s digests", imageName) + } + if imageDigests != checkSum { + logrus.Errorln("checkSumFailed ", imageDigests, " mismatch to ", checkSum) + return fmt.Errorf("checkSumFailed %s mismatch to %s", imageDigests, checkSum) + } + return nil +} + +func deepCopyConfigMap(m map[string]*pb.KeyInfo) map[string]*pb.KeyInfo { + result := make(map[string]*pb.KeyInfo) + for key, val := range m { + result[key] = &pb.KeyInfo{ + Value: val.Value, + Operation: val.Operation, + } + } + return result +} + +func isCommandAvailable(name string) bool { + _, err := exec.LookPath(name) + return err == nil +} + +func isValidImageName(image string) error { + pattern := `^((?:[\w.-]+)(?::\d+)?\/)*(?:[\w.-]+)((?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$` + regEx, err := regexp.Compile(pattern) + if err != nil { + return err + } + if !regEx.MatchString(image) { + return fmt.Errorf("invalid image name %s", image) + } + return nil +} + +func getOCIImageDigest(containerRuntime string, imageName string) (string, error) { var cmdOutput string var err error + var imageDigests string switch containerRuntime { case "crictl": cmdOutput, err = runCommandWithOut("crictl", "inspecti", "--output", "go-template", "--template", "{{.status.repoDigests}}", imageName) if err != nil { - return err + return "", err } case "docker": cmdOutput, err = runCommandWithOut("docker", "inspect", "--format", "{{.RepoDigests}}", imageName) if err != nil { - return err + return "", err } case "ctr": cmdOutput, err = runCommandWithOut("ctr", "-n", "k8s.io", "images", "ls", "name=="+imageName) if err != nil { - return err + return "", err } // after Fields, we get slice like [REF TYPE DIGEST SIZE PLATFORMS LABELS x x x x x x] // the digest is the position 8 element imageDigest := strings.Split(strings.Fields(cmdOutput)[8], ":")[1] - if imageDigest != checkSum { - logrus.Errorln("checkSumFailed ", imageDigest, " mismatch to ", checkSum) - return fmt.Errorf("checkSumFailed %s mismatch to %s", imageDigest, checkSum) - } - return nil + return imageDigest, nil default: logrus.Errorln("containerRuntime ", containerRuntime, " cannot be recognized") - return fmt.Errorf("containerRuntime %s cannot be recognized", containerRuntime) + return "", fmt.Errorf("containerRuntime %s cannot be recognized", containerRuntime) } // cmdOutput format is as follows: // [imageRepository/imageName:imageTag@sha256:digests] // parse the output and get digest - var imageDigests string outArray := strings.Split(cmdOutput, "@") if strings.HasPrefix(outArray[len(outArray)-1], "sha256") { pasredArray := strings.Split(strings.TrimSuffix(outArray[len(outArray)-1], "]"), ":") @@ -329,41 +369,12 @@ func checkOCIImageDigestMatch(containerRuntime string, imageName string, checkSu imageDigests = pasredArray[digestIndex] } } - if imageDigests == "" { - logrus.Errorln("error when get ", imageName, " digests") - return fmt.Errorf("error when get %s digests", imageName) - } - if imageDigests != checkSum { - logrus.Errorln("checkSumFailed ", imageDigests, " mismatch to ", checkSum) - return fmt.Errorf("checkSumFailed %s mismatch to %s", imageDigests, checkSum) - } - return nil + return imageDigests, nil } -func deepCopyConfigMap(m map[string]*pb.KeyInfo) map[string]*pb.KeyInfo { - result := make(map[string]*pb.KeyInfo) - for key, val := range m { - result[key] = &pb.KeyInfo{ - Value: val.Value, - Operation: val.Operation, - } - } - return result -} - -func isCommandAvailable(name string) bool { - _, err := exec.LookPath(name) - return err == nil -} - -func isValidImageName(image string) error { - pattern := `^((?:[\w.-]+)(?::\d+)?\/)*(?:[\w.-]+)(?::[\w_.-]+)?(?:@sha256:[a-fA-F0-9]+)?$` - regEx, err := regexp.Compile(pattern) - if err != nil { +func modifyImageLabel(imagePath, side, next string) error { + if err := runCommand("e2label", imagePath, "ROOT-"+next); err != nil { return err } - if !regEx.MatchString(image) { - return fmt.Errorf("invalid image name %s", image) - } return nil } diff --git a/cmd/agent/server/utils_test.go b/cmd/agent/server/utils_test.go index 8e7fd9089fcb59cdf384b0be0a5af5f7e9713623..da53c0e6d9bac0a5289ffa44cb62b63ee252fcd8 100644 --- a/cmd/agent/server/utils_test.go +++ b/cmd/agent/server/utils_test.go @@ -14,13 +14,18 @@ package server import ( + "archive/tar" + "fmt" "os" "os/exec" - "strings" + "reflect" "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" ) -func TestrunCommand(t *testing.T) { +func Test_runCommand(t *testing.T) { type args struct { name string args []string @@ -41,23 +46,29 @@ func TestrunCommand(t *testing.T) { } } -func Testinstall(t *testing.T) { +func Test_install(t *testing.T) { type args struct { imagePath string side string next string } - out, _ := exec.Command("bash", "-c", "df -h | grep '/$' | awk '{print $1}'").CombinedOutput() - mountPart := strings.TrimSpace(string(out)) tests := []struct { name string args args wantErr bool }{ - {name: "normal", args: args{imagePath: "aa.txt", side: mountPart, next: ""}, wantErr: false}, + {name: "normal uefi", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: false}, + {name: "normal legacy", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: false}, + {name: "get boot mode error", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: true}, } - ff, _ := os.Create("aa.txt") - ff.Chmod(os.ModePerm) + patchRunCommand := gomonkey.ApplyFuncReturn(runCommand, nil) + defer patchRunCommand.Reset() + patchGetBootMode := gomonkey.ApplyFuncSeq(getBootMode, []gomonkey.OutputCell{ + {Values: gomonkey.Params{"uefi", nil}}, + {Values: gomonkey.Params{"legacy", nil}}, + {Values: gomonkey.Params{"", fmt.Errorf("error")}}, + }) + defer patchGetBootMode.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := install(tt.args.imagePath, tt.args.side, tt.args.next); (err != nil) != tt.wantErr { @@ -65,17 +76,13 @@ func Testinstall(t *testing.T) { } }) } - ff.Close() - defer os.Remove("aa.txt") } -func TestgetNextPart(t *testing.T) { +func Test_getNextPart(t *testing.T) { type args struct { partA string partB string } - out, _ := exec.Command("bash", "-c", "df -h | grep '/$' | awk '{print $1}'").CombinedOutput() - mountPart := strings.TrimSpace(string(out)) tests := []struct { name string args args @@ -83,8 +90,16 @@ func TestgetNextPart(t *testing.T) { want1 string wantErr bool }{ - {name: "normal", args: args{partA: mountPart, partB: "testB"}, want: "testB", want1: "B", wantErr: false}, + {name: "switch to sda3", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "/dev/sda3", want1: "B", wantErr: false}, + {name: "switch to sda2", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "/dev/sda2", want1: "A", wantErr: false}, + {name: "error", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "", want1: "", wantErr: true}, } + patchExecCommand := gomonkey.ApplyMethodSeq(&exec.Cmd{}, "CombinedOutput", []gomonkey.OutputCell{ + {Values: gomonkey.Params{[]byte("/"), nil}}, + {Values: gomonkey.Params{[]byte(""), nil}}, + {Values: gomonkey.Params{[]byte(""), fmt.Errorf("error")}}, + }) + defer patchExecCommand.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, got1, err := getNextPart(tt.args.partA, tt.args.partB) @@ -101,3 +116,325 @@ func TestgetNextPart(t *testing.T) { }) } } + +func Test_prepareEnv(t *testing.T) { + mountPath := "/persist/KubeOS-Update/kubeos-update" + if err := os.MkdirAll(mountPath, 0644); err != nil { + t.Fatalf("mkdir err %v", err) + } + defer os.RemoveAll("/persist") + tests := []struct { + name string + want preparePath + wantErr bool + }{ + { + name: "success", + want: preparePath{ + updatePath: "/persist/KubeOS-Update", + mountPath: "/persist/KubeOS-Update/kubeos-update", + tarPath: "/persist/KubeOS-Update/os.tar", + imagePath: "/persist/update.img", + rootfsFile: "os.tar", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := prepareEnv() + if (err != nil) != tt.wantErr { + t.Errorf("prepareEnv() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("prepareEnv() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_createOSImage(t *testing.T) { + mountPath := "/persist/KubeOS-Update/kubeos-update" + if err := os.MkdirAll(mountPath, 0644); err != nil { + t.Fatalf("mkdir err %v", err) + } + defer os.RemoveAll("/persist") + tarPath := "/persist/KubeOS-Update/os.tar" + path, err := createTmpTarFile(tarPath) + if path != tarPath && err != nil { + t.Fatalf("create temp zip file err %v", err) + } + type args struct { + neededPath preparePath + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "normal", + args: args{ + neededPath: preparePath{ + updatePath: "/persist/KubeOS-Update", + mountPath: "/persist/KubeOS-Update/kubeos-update", + tarPath: "/persist/KubeOS-Update/os.tar", + imagePath: "/persist/update.img", + }, + }, + want: "/persist/update.img", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := createOSImage(tt.args.neededPath) + if (err != nil) != tt.wantErr { + t.Errorf("createOSImage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("createOSImage() = %v, want %v", got, tt.want) + } + }) + } +} + +func createTmpTarFile(tarPath string) (string, error) { + tempFile, err := os.Create(tarPath) + if err != nil { + return "", err + } + defer tempFile.Close() + + tarWriter := tar.NewWriter(tempFile) + fakeData := []byte("This is a fake file") + fakeFile := "fakefile.txt" + header := &tar.Header{ + Name: fakeFile, + Size: int64(len(fakeData)), + Mode: 0644, + ModTime: time.Now(), + } + + if err = tarWriter.WriteHeader(header); err != nil { + return "", err + } + if _, err := tarWriter.Write(fakeData); err != nil { + return "", err + } + if err := tarWriter.Flush(); err != nil { + return "", err + } + return tempFile.Name(), nil +} + +func Test_getBootMode(t *testing.T) { + tests := []struct { + name string + want string + wantErr bool + }{ + { + name: "uefi", + want: "uefi", + wantErr: false, + }, + { + name: "legacy", + want: "legacy", + wantErr: false, + }, + { + name: "error", + want: "", + wantErr: true, + }, + } + patchOSStat := gomonkey.ApplyFuncSeq(os.Stat, []gomonkey.OutputCell{ + {Values: gomonkey.Params{nil, nil}}, + {Values: gomonkey.Params{nil, os.ErrNotExist}}, + {Values: gomonkey.Params{nil, fmt.Errorf("fake error")}}, + }) + defer patchOSStat.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getBootMode() + if (err != nil) != tt.wantErr { + t.Errorf("getBootMode() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getBootMode() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_isValidImageName(t *testing.T) { + type args struct { + image string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "valid", args: args{image: "alpine"}, wantErr: false}, + {name: "valid", args: args{image: "alpine:latest"}, wantErr: false}, + {name: "valid", args: args{image: "localhost:1234/test"}, wantErr: false}, + {name: "valid", args: args{image: "alpine:3.7"}, wantErr: false}, + {name: "valid", args: args{image: "docker.example.edu/gmr/alpine:3.7"}, wantErr: false}, + {name: "valid", args: args{image: "docker.example.com:5000/gmr/alpine@sha256:11111111111111111111111111111111"}, wantErr: false}, + {name: "valid", args: args{image: "registry.dobby.org/dobby/dobby-servers/arthound:2019-08-08"}, wantErr: false}, + {name: "valid", args: args{image: "registry.dobby.org/dobby/dobby-servers/lerphound:latest"}, wantErr: false}, + {name: "valid", args: args{image: "registry.dobby.org/dobby/dobby-servers/loophole@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8"}, wantErr: false}, + {name: "valid", args: args{image: "sosedoff/pgweb@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8"}, wantErr: false}, + {name: "valid", args: args{image: "registry.dobby.org/dobby/antique-penguin:release-production"}, wantErr: false}, + {name: "valid", args: args{image: "dalprodictus/halcon:6.7.5"}, wantErr: false}, + {name: "valid", args: args{image: "antigua/antigua:v31"}, wantErr: false}, + {name: "invalid ;", args: args{image: "alpine;v1.0"}, wantErr: true}, + {name: "invalid tag and digest1", args: args{image: "alpine:latest@sha256:11111111111111111111111111111111"}, wantErr: true}, + {name: "invalid |", args: args{image: "alpine|v1.0"}, wantErr: true}, + {name: "invalid &", args: args{image: "alpine&v1.0"}, wantErr: true}, + {name: "invalid tag and digest2", args: args{image: "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8"}, wantErr: true}, + {name: "invalid tag and digest3", args: args{image: "192.168.122.123:5000/kubeos_uefi-x86_64:euleros_v2_docker-2023-01@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := isValidImageName(tt.args.image); (err != nil) != tt.wantErr { + t.Errorf("isValidImageName() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_checkOCIImageDigestMatch(t *testing.T) { + type args struct { + containerRuntime string + imageName string + checkSum string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "invalid container runtion", args: args{containerRuntime: "dockctl", imageName: "docker.io/library/hello-world:latest", checkSum: "1abf18abf9bf9baa0a4a38d1afad4abf0d7da4544e163186e036c906c09c94fe"}, wantErr: true}, + {name: "nil image digets", args: args{containerRuntime: "crictl", imageName: "docker.io/library/hello-world:latest", checkSum: "1abf18abf9bf9baa0a4a38d1afad4abf0d7da4544e163186e036c906c09c94fe"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.name == "nil image digets" { + patchGetOCIImageDigest := gomonkey.ApplyFuncReturn(getOCIImageDigest, "", nil) + defer patchGetOCIImageDigest.Reset() + } + if err := checkOCIImageDigestMatch(tt.args.containerRuntime, tt.args.imageName, tt.args.checkSum); (err != nil) != tt.wantErr { + t.Errorf("checkOCIImageDigestMatch() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_runCommandWithOut(t *testing.T) { + type args struct { + name string + args []string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "error", args: args{name: "/mmm", args: []string{"", ""}}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := runCommandWithOut(tt.args.name, tt.args.args...) + if (err != nil) != tt.wantErr { + t.Errorf("runCommandWithOut() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("runCommandWithOut() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getRootfsDisks(t *testing.T) { + tests := []struct { + name string + want string + want1 string + wantErr bool + }{ + {name: "error", want: "", want1: "", wantErr: true}, + } + patchRunCommandWithOut := gomonkey.ApplyFuncSeq(runCommandWithOut, []gomonkey.OutputCell{ + {Values: gomonkey.Params{"", fmt.Errorf("fake error")}}, + }) + defer patchRunCommandWithOut.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := getRootfsDisks() + if (err != nil) != tt.wantErr { + t.Errorf("getRootfsDisks() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getRootfsDisks() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("getRootfsDisks() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_checkDiskSize(t *testing.T) { + type args struct { + needGBSize int + path string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "zero GB need", args: args{needGBSize: 0, path: "/dev/sda"}, wantErr: false}, + {name: "disk not enough", args: args{needGBSize: 100000, path: "/dev/sda"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := checkDiskSize(tt.args.needGBSize, tt.args.path); (err != nil) != tt.wantErr { + t.Errorf("checkDiskSize() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_deleteFile(t *testing.T) { + type args struct { + path string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "error", args: args{path: "/mmm"}, wantErr: true}, + } + patchStat := gomonkey.ApplyFuncReturn(os.Stat, nil, fmt.Errorf("fake error")) + defer patchStat.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := deleteFile(tt.args.path); (err != nil) != tt.wantErr { + t.Errorf("deleteFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go new file mode 100644 index 0000000000000000000000000000000000000000..4b441d18e72142c69891c62e6a6530009090f05c --- /dev/null +++ b/cmd/operator/controllers/operation.go @@ -0,0 +1,181 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package controllers contains the Reconcile of operator +package controllers + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + upgradev1 "openeuler.org/KubeOS/api/v1alpha1" + "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type operation interface { + newExistRequirement() (labels.Requirement, error) + newNotExistRequirement() (labels.Requirement, error) + updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) + updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error +} + +type upgradeOps struct{} + +func (u upgradeOps) newExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (u upgradeOps) newNotExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) { + var count int + for _, node := range nodes { + if count >= limit { + break + } + osVersionNode := node.Status.NodeInfo.OSImage + if os.Spec.OSVersion != osVersionNode { + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "failed to get osInstance "+node.Name, "skip this node") + return count, err + } + continue + } + if err := u.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + continue + } + count++ + } + } + return count, nil +} + +func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error { + if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { + if err := deepCopySpecConfigs(os, osInstance, values.UpgradeConfigName); err != nil { + return err + } + } + if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { + if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { + return err + } + // exchange "grub.cmdline.current" and "grub.cmdline.next" + for i, config := range osInstance.Spec.SysConfigs.Configs { + if config.Model == "grub.cmdline.current" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" + } + if config.Model == "grub.cmdline.next" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" + } + } + } + osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } + node.Labels[values.LabelUpgrading] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } + return nil +} + +type configOps struct{} + +func (c configOps) newExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.Exists, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelConfiguring) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (c configOps) newNotExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.DoesNotExist, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelConfiguring) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) { + var count int + for _, node := range nodes { + if count >= limit { + break + } + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "failed to get osInstance "+node.Name) + return count, err + } + continue + } + if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { + if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + continue + } + count++ + } + } + return count, nil +} + +func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error { + if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { + return err + } + osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } + node.Labels[values.LabelConfiguring] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } + return nil +} diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index f86a0b2c44b81c57a6c483dbf1404aa50b6efe5c..bd7a70dd7230ef82571834ee946be5d9addccc8f 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -15,6 +15,9 @@ package controllers import ( "context" + "encoding/json" + "fmt" + "reflect" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -63,29 +66,24 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) } ops := os.Spec.OpsType + var opsInsatnce operation switch ops { case "upgrade", "rollback": - limit, err := checkUpgrading(ctx, r, min(os.Spec.MaxUnavailable, nodeNum)) // adjust maxUnavailable if need - if err != nil { - return values.RequeueNow, err - } - if needRequeue, err := assignUpgrade(ctx, r, os, limit, req.Namespace); err != nil { - return values.RequeueNow, err - } else if needRequeue { - return values.Requeue, nil - } + opsInsatnce = upgradeOps{} case "config": - limit, err := checkConfig(ctx, r, min(os.Spec.MaxUnavailable, nodeNum)) - if err != nil { - return values.RequeueNow, err - } - if needRequeue, err := assignConfig(ctx, r, os.Spec.SysConfigs, os.Spec.SysConfigs.Version, limit); err != nil { - return values.RequeueNow, err - } else if needRequeue { - return values.Requeue, nil - } + opsInsatnce = configOps{} default: log.Error(nil, "operation "+ops+" cannot be recognized") + return values.Requeue, nil + } + limit, err := calNodeLimit(ctx, r, opsInsatnce, min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need + if err != nil { + return values.RequeueNow, err + } + if needRequeue, err := assignOperation(ctx, r, os, limit, opsInsatnce); err != nil { + return values.RequeueNow, err + } else if needRequeue { + return values.Requeue, nil } return values.Requeue, nil } @@ -94,7 +92,11 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OSInstance{}, values.OsiStatusName, func(rawObj client.Object) []string { - osi := rawObj.(*upgradev1.OSInstance) + osi, ok := rawObj.(*upgradev1.OSInstance) + if !ok { + log.Error(nil, "failed to convert to osInstance") + return []string{} + } return []string{osi.Spec.NodeStatus} }); err != nil { return err @@ -109,7 +111,7 @@ func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLimitingInterface) { ctx := context.Background() hostname := e.Object.GetName() - labelSelector := labels.SelectorFromSet(labels.Set{"upgrade.openeuler.org/osinstance-node": hostname}) + labelSelector := labels.SelectorFromSet(labels.Set{values.LabelOSinstance: hostname}) osInstanceList := &upgradev1.OSInstanceList{} if err := r.List(ctx, osInstanceList, client.MatchingLabelsSelector{Selector: labelSelector}); err != nil { log.Error(err, "unable to list osInstances") @@ -123,30 +125,44 @@ func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLim } } -func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (os upgradev1.OS, - nodeNum int, err error) { - if err = r.Get(ctx, name, &os); err != nil { +func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, + int, error) { + var os upgradev1.OS + if err := r.Get(ctx, name, &os); err != nil { log.Error(err, "unable to fetch OS") - return + return upgradev1.OS{}, 0, err } requirement, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) if err != nil { log.Error(err, "unable to create requirement "+values.LabelMaster) - return + return upgradev1.OS{}, 0, err } - nodesItems, err := getNodes(ctx, r, 0, *requirement) + var requirements []labels.Requirement + requirements = append(requirements, *requirement) + if os.Spec.NodeSelector != "" { + reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Exists, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return upgradev1.OS{}, 0, err + } + requirements = append(requirements, *requirement, *reqSelector) + } + nodesItems, err := getNodes(ctx, r, 0, requirements...) if err != nil { log.Error(err, "get slave nodes fail") - return + return upgradev1.OS{}, 0, err } - nodeNum = len(nodesItems) - return + nodeNum := len(nodesItems) + return os, nodeNum, nil } -func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, - nameSpace string) (bool, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) +func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, + ops operation) (bool, error) { + fmt.Println("start assignOperation") + fmt.Println("ops is ", reflect.TypeOf(ops)) + requirement, err := ops.newNotExistRequirement() + fmt.Println("requirement is ", requirement.String()) if err != nil { log.Error(err, "unable to create requirement "+values.LabelUpgrading) return false, err @@ -156,80 +172,29 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. log.Error(err, "unable to create requirement "+values.LabelMaster) return false, err } - - nodes, err := getNodes(ctx, r, limit+1, *requirement, *reqMaster) // one more to see if all node updated - if err != nil { - return false, err - } - - var count = 0 - for _, node := range nodes { - if count >= limit { - break - } - osVersionNode := node.Status.NodeInfo.OSImage - if os.Spec.OSVersion != osVersionNode { - var osInstance upgradev1.OSInstance - if err = r.Get(ctx, types.NamespacedName{Namespace: nameSpace, Name: node.Name}, &osInstance); err != nil { - if err = client.IgnoreNotFound(err); err != nil { - log.Error(err, "failed to get osInstance "+node.Name) - return false, err - } - continue - } - count++ - node.Labels[values.LabelUpgrading] = "" - expUpVersion := os.Spec.UpgradeConfigs.Version - osiUpVersion := osInstance.Spec.UpgradeConfigs.Version - if osiUpVersion != expUpVersion { - osInstance.Spec.UpgradeConfigs = os.Spec.UpgradeConfigs - } - expSysVersion := os.Spec.SysConfigs.Version - osiSysVersion := osInstance.Spec.SysConfigs.Version - if osiSysVersion != expSysVersion { - osInstance.Spec.SysConfigs = os.Spec.SysConfigs - for i, config := range osInstance.Spec.SysConfigs.Configs { - if config.Model == "grub.cmdline.current" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" - } - if config.Model == "grub.cmdline.next" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" - } - } - } - osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() - if err = r.Update(ctx, &osInstance); err != nil { - log.Error(err, "unable to update", "osInstance", osInstance.Name) - } - if err = r.Update(ctx, &node); err != nil { - log.Error(err, "unable to label", "node", node.Name) - } + var requirements []labels.Requirement + requirements = append(requirements, requirement, *reqMaster) + if os.Spec.NodeSelector != "" { + reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{os.Spec.NodeSelector}) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return false, err } + fmt.Println("requirement is ", reqSelector.String()) + requirements = append(requirements, *reqSelector) } - return count >= limit, nil -} -func assignConfig(ctx context.Context, r common.ReadStatusWriter, sysConfigs upgradev1.SysConfigs, - configVersion string, limit int) (bool, error) { - osInstances, err := getIdleOSInstances(ctx, r, limit+1) // one more to see if all node updated + nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated if err != nil { return false, err } - var count = 0 - for _, osInstance := range osInstances { - if count >= limit { - break - } - configVersionNode := osInstance.Spec.SysConfigs.Version - if configVersion != configVersionNode { - count++ - osInstance.Spec.SysConfigs = sysConfigs - osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() - if err = r.Update(ctx, &osInstance); err != nil { - log.Error(err, "unable update osInstance ", "osInstanceName ", osInstance.Name) - } - } + fmt.Println("nodes has not upgrade/config and has selector ", len(nodes)) + // Upgrade OS for selected nodes + count, err := ops.updateNodes(ctx, r, &os, nodes, limit) + if err != nil { + return false, err } + return count >= limit, nil } @@ -244,48 +209,37 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, return nodeList.Items, nil } -func getIdleOSInstances(ctx context.Context, r common.ReadStatusWriter, limit int) ([]upgradev1.OSInstance, error) { - var osInstanceList upgradev1.OSInstanceList - opt := []client.ListOption{ - client.MatchingFields{values.OsiStatusName: values.NodeStatusIdle.String()}, - &client.ListOptions{Limit: int64(limit)}, - } - if err := r.List(ctx, &osInstanceList, opt...); err != nil { - log.Error(err, "unable to list nodes with requirements") - return nil, err - } - return osInstanceList.Items, nil -} - -func getConfigOSInstances(ctx context.Context, r common.ReadStatusWriter) ([]upgradev1.OSInstance, error) { - var osInstanceList upgradev1.OSInstanceList - if err := r.List(ctx, &osInstanceList, - client.MatchingFields{values.OsiStatusName: values.NodeStatusConfig.String()}); err != nil { - log.Error(err, "unable to list nodes with requirements") - return nil, err - } - return osInstanceList.Items, nil -} - -func checkUpgrading(ctx context.Context, r common.ReadStatusWriter, maxUnavailable int) (int, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) +func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, + ops operation, maxUnavailable int, nodeSelector string) (int, error) { + fmt.Println("start calNodeLimit") + fmt.Println("ops is ", reflect.TypeOf(ops)) + requirement, err := ops.newExistRequirement() if err != nil { log.Error(err, "unable to create requirement "+values.LabelUpgrading) return 0, err } - nodes, err := getNodes(ctx, r, 0, *requirement) - if err != nil { - return 0, err + fmt.Println("requirement is ", requirement.String()) + var requirements []labels.Requirement + requirements = append(requirements, requirement) + if nodeSelector != "" { + reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{nodeSelector}) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return 0, err + } + fmt.Println("requirement is ", reqSelector.String()) + requirements = append(requirements, *reqSelector) } - return maxUnavailable - len(nodes), nil -} - -func checkConfig(ctx context.Context, r common.ReadStatusWriter, maxUnavailable int) (int, error) { - osInstances, err := getConfigOSInstances(ctx, r) + nodes, err := getNodes(ctx, r, 0, requirements...) if err != nil { return 0, err + } - return maxUnavailable - len(osInstances), nil + fmt.Println("nodes has upgrade and selector ", len(nodes)) + for _, n := range nodes { + fmt.Println(" nodes name is ", n.Name) + } + return maxUnavailable - len(nodes), nil } func min(a, b int) int { @@ -294,3 +248,28 @@ func min(a, b int) int { } return b } + +func deepCopySpecConfigs(os *upgradev1.OS, osinstance *upgradev1.OSInstance, configType string) error { + switch configType { + case values.UpgradeConfigName: + data, err := json.Marshal(os.Spec.UpgradeConfigs) + if err != nil { + return err + } + if err = json.Unmarshal(data, &osinstance.Spec.UpgradeConfigs); err != nil { + return err + } + case values.SysConfigName: + data, err := json.Marshal(os.Spec.SysConfigs) + if err != nil { + return err + } + if err = json.Unmarshal(data, &osinstance.Spec.SysConfigs); err != nil { + return err + } + default: + log.Error(nil, "configType "+configType+" cannot be recognized") + return fmt.Errorf("configType %s cannot be recognized", configType) + } + return nil +} diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index 30a773ca63798f815e786147055729545c3e0dd1..98afb26cc13a32616923fce734fc2bd8eaf3cb66 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -14,25 +14,28 @@ package controllers import ( "context" + "fmt" + "reflect" + "testing" "time" + "github.com/agiledragon/gomonkey/v2" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - upgradev1 "openeuler.org/KubeOS/api/v1alpha1" + "openeuler.org/KubeOS/pkg/common" "openeuler.org/KubeOS/pkg/values" ) var _ = Describe("OsController", func() { const ( - OSName = "test-os" - + OSName = "test-os" timeout = time.Second * 20 interval = time.Millisecond * 500 ) @@ -66,26 +69,33 @@ var _ = Describe("OsController", func() { }) AfterEach(func() { - desiredTestNamespace := &v1.Namespace{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Namespace", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: testNamespace, - }, + // delete all nodes + nodeList := &v1.NodeList{} + err := k8sClient.List(context.Background(), nodeList) + Expect(err).ToNot(HaveOccurred()) + for _, node := range nodeList.Items { + k8sClient.Delete(context.Background(), &node) } - // Add any teardown steps that needs to be executed after each test - err := k8sClient.Delete(context.Background(), desiredTestNamespace, - client.PropagationPolicy(metav1.DeletePropagationForeground)) + nodeList = &v1.NodeList{} + Eventually(func() bool { + err = k8sClient.List(context.Background(), nodeList) + if err != nil || len(nodeList.Items) != 0 { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + // delete all OS CRs + osList := &upgradev1.OSList{} + err = k8sClient.List(context.Background(), osList) Expect(err).ToNot(HaveOccurred()) - - existingNamespace := &v1.Namespace{} + for _, os := range osList.Items { + k8sClient.Delete(context.Background(), &os) + } + osList = &upgradev1.OSList{} Eventually(func() bool { - err := k8sClient.Get(context.Background(), types.NamespacedName{Name: testNamespace}, - existingNamespace) - if err != nil && errors.IsNotFound(err) { + err = k8sClient.List(context.Background(), osList) + if err != nil || len(osList.Items) != 0 { return false } return true @@ -96,7 +106,7 @@ var _ = Describe("OsController", func() { It("Should label the osinstance's nodestatus to upgrading", func() { ctx := context.Background() - // create Node + // create Node1 node1Name = "test-node-" + uuid.New().String() node1 := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -125,7 +135,7 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) - // create OSInstance + // create OSInstance1 OSIns := &upgradev1.OSInstance{ TypeMeta: metav1.TypeMeta{ Kind: "OSInstance", @@ -134,6 +144,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ SysConfigs: upgradev1.SysConfigs{ @@ -154,6 +167,67 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v2", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + // create OS CR OS := &upgradev1.OS{ TypeMeta: metav1.TypeMeta{ @@ -189,7 +263,7 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -197,13 +271,20 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + + createdOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) }) }) Context("When we want to configure node", func() { It("Should update OSInstance spec and update NodeStatus to config", func() { ctx := context.Background() - // create Node + // create Node1 node1Name = "test-node-" + uuid.New().String() node1 := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -219,7 +300,7 @@ var _ = Describe("OsController", func() { }, Status: v1.NodeStatus{ NodeInfo: v1.NodeSystemInfo{ - OSImage: "KubeOS v2", + OSImage: "KubeOS v1", }, }, } @@ -232,6 +313,7 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) + // create OSInstance1 OSIns := &upgradev1.OSInstance{ TypeMeta: metav1.TypeMeta{ Kind: "OSInstance", @@ -240,6 +322,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ SysConfigs: upgradev1.SysConfigs{ @@ -252,14 +337,76 @@ var _ = Describe("OsController", func() { } Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) - osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns := &upgradev1.OSInstance{} Eventually(func() bool { - err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) return err == nil }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + OS := &upgradev1.OS{ TypeMeta: metav1.TypeMeta{ APIVersion: "upgrade.openeuler.org/v1alpha1", @@ -301,14 +448,38 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished - configedOSIns := &upgradev1.OSInstance{} + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + configedOSIns1 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, configedOSIns1) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(configedOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) + Expect(configedOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + existingNode1 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode1) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode1.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(true)) + + configedOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, configedOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(configedOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) + Expect(configedOSIns2.Spec.SysConfigs.Version).Should(Equal("v2")) + existingNode2 := &v1.Node{} Eventually(func() bool { - err := k8sClient.Get(ctx, osInsCRLookupKey, configedOSIns) + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) return err == nil }, timeout, interval).Should(BeTrue()) - Expect(configedOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) - Expect(configedOSIns.Spec.SysConfigs.Version).Should(Equal("v2")) + _, ok = existingNode2.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(true)) }) }) @@ -385,7 +556,760 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) _, ok := existingNode.Labels[values.LabelUpgrading] Expect(ok).Should(Equal(false)) + + createdOS.Spec.OpsType = "test" + Expect(k8sClient.Update(ctx, createdOS)).Should(Succeed()) }) }) -}) + Context("When we want to upgrade and do sysconfigs which contain grub.cmd.current and .next", func() { + It("Should exchange .current and .next", func() { + ctx := context.Background() + + // create Node1 + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance1 + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + + // create OS CR + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "upgrade", + MaxUnavailable: 3, + OSVersion: "KubeOS v2", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + {Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}}, + {Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}}, + }, + }, + UpgradeConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + {Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}}, + {Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}}, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + // Check that the corresponding OS CR has been created + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + // check node1 osinstance + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.Spec.SysConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + Expect(createdOSIns.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) + Expect(createdOSIns.Spec.UpgradeConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + + // check node2 osinstance + createdOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + Expect(createdOSIns2.Spec.SysConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + Expect(createdOSIns2.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) + Expect(createdOSIns2.Spec.UpgradeConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + + // check os cr spec + osCRLookupKey = types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS = &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.SysConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + Expect(createdOS.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) + }) + }) + + Context("When we want to upgrade node with nodes having NodeSelector label", func() { + It("Should only update node with NodeSelector label", func() { + ctx := context.Background() + // create Node1 + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + "upgrade.openeuler.org/node-selector": "openeuler", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance1 + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "upgrade", + MaxUnavailable: 3, + OSVersion: "KubeOS v2", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + NodeSelector: "openeuler", + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }, + }, + }, + UpgradeConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + {Model: "kernel.sysctl.persist", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }}, + }, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + existingNode1 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode1) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode1.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(true)) + + upgradeOSIns1 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, upgradeOSIns1) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + Expect(upgradeOSIns1.Spec.UpgradeConfigs.Version).Should(Equal("v2")) + Expect(upgradeOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + + existingNode2 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok = existingNode2.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(false)) + + upgradeOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, upgradeOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + Expect(upgradeOSIns2.Spec.UpgradeConfigs.Version).Should(Equal("v1")) + Expect(upgradeOSIns2.Spec.SysConfigs.Version).Should(Equal("v1")) + }) + }) + + Context("When we want to config node with nodes having NodeSelector label", func() { + It("Should only config node with NodeSelector label", func() { + ctx := context.Background() + // create Node1 + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + "upgrade.openeuler.org/node-selector": "openeuler", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance1 + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "config", + MaxUnavailable: 3, + OSVersion: "KubeOS v1", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + NodeSelector: "openeuler", + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.SysConfigs.Version).Should(Equal("v2")) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + existingNode1 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode1) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode1.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(true)) + + upgradeOSIns1 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, upgradeOSIns1) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) + Expect(upgradeOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + + existingNode2 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok = existingNode2.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(false)) + + upgradeOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, upgradeOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + Expect(upgradeOSIns2.Spec.SysConfigs.Version).Should(Equal("v1")) + }) + }) +}) + +func Test_deepCopySpecConfigs(t *testing.T) { + type args struct { + os *upgradev1.OS + osinstance *upgradev1.OSInstance + configType string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "error", + args: args{ + os: &upgradev1.OS{}, + osinstance: &upgradev1.OSInstance{}, + configType: "test"}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := deepCopySpecConfigs(tt.args.os, tt.args.osinstance, tt.args.configType); (err != nil) != tt.wantErr { + t.Errorf("deepCopySpecConfigs() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_getNodes(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + limit int + reqs []labels.Requirement + } + tests := []struct { + name string + args args + want []corev1.Node + wantErr bool + }{ + { + name: "list error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + limit: 1, + }, + want: nil, + wantErr: true, + }, + } + patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ + {Values: gomonkey.Params{fmt.Errorf("list error")}}, + }) + defer patchList.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getNodes(tt.args.ctx, tt.args.r, tt.args.limit, tt.args.reqs...) + if (err != nil) != tt.wantErr { + t.Errorf("getNodes() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getNodes() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getAndUpdateOS(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + name types.NamespacedName + } + tests := []struct { + name string + args args + wantOs upgradev1.OS + wantNodeNum int + wantErr bool + }{ + { + name: "label error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + name: types.NamespacedName{Namespace: "test_ns", Name: "test"}, + }, + wantOs: upgradev1.OS{}, + wantNodeNum: 0, + wantErr: true, + }, + { + name: "get nodes error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + name: types.NamespacedName{Namespace: "test_ns", Name: "test"}, + }, + wantOs: upgradev1.OS{}, + wantNodeNum: 0, + wantErr: true, + }, + } + patchGet := gomonkey.ApplyMethodReturn(&OSReconciler{}, "Get", nil) + defer patchGet.Reset() + patchNewRequirement := gomonkey.ApplyFuncSeq(labels.NewRequirement, []gomonkey.OutputCell{ + {Values: gomonkey.Params{nil, fmt.Errorf("label error")}}, + {Values: gomonkey.Params{&labels.Requirement{}, nil}}, + }) + defer patchNewRequirement.Reset() + patchGetNodes := gomonkey.ApplyFuncReturn(getNodes, nil, fmt.Errorf("get nodes error")) + defer patchGetNodes.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotOs, gotNodeNum, err := getAndUpdateOS(tt.args.ctx, tt.args.r, tt.args.name) + if (err != nil) != tt.wantErr { + t.Errorf("getAndUpdateOS() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotOs, tt.wantOs) { + t.Errorf("getAndUpdateOS() gotOs = %v, want %v", gotOs, tt.wantOs) + } + if gotNodeNum != tt.wantNodeNum { + t.Errorf("getAndUpdateOS() gotNodeNum = %v, want %v", gotNodeNum, tt.wantNodeNum) + } + }) + } +} diff --git a/cmd/operator/controllers/suite_test.go b/cmd/operator/controllers/suite_test.go index 889789ebed7ee2b0fa6147a12319c23eb4d6fa90..aa6deeae581a377285d4fe43b19a51b32138b93b 100644 --- a/cmd/operator/controllers/suite_test.go +++ b/cmd/operator/controllers/suite_test.go @@ -51,7 +51,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "docs", "example", "config", "crd")}, ErrorIfCRDPathMissing: true, } diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 17b74e1b3d22e2353869d1ecb1d4b6844c1b1c62..8249ad2d40f2ba337072a2cc1faf41edbea70ee5 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -41,7 +41,11 @@ func init() { } func main() { - mgr := common.NewControllerManager(setupLog, scheme) + mgr, err := common.NewControllerManager(setupLog, scheme) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } if err := (&controllers.OSReconciler{ Client: mgr.GetClient(), diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index a17afacfd6796949350720f557994301f2687d9e..b543befc7bb8be213b7f11a4cfaf4c4fab661af6 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -87,18 +87,21 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re sameOSVersion := checkVersion(osCr.Spec.OSVersion, node.Status.NodeInfo.OSImage) if sameOSVersion { configOps, err := checkConfigVersion(osCr, osInstance, values.SysConfigName) + if err != nil { + return values.RequeueNow, err + } if configOps == values.Reassign { if err = r.refreshNode(ctx, &node, osInstance, osCr.Spec.SysConfigs.Version, values.SysConfigName); err != nil { return values.RequeueNow, err } - return values.RequeueNow, nil + return values.Requeue, nil } if configOps == values.UpdateConfig { osInstance.Spec.SysConfigs = osCr.Spec.SysConfigs if err = r.Update(ctx, osInstance); err != nil { return values.RequeueNow, err } - return values.RequeueNow, nil + return values.Requeue, nil } if err := r.setConfig(ctx, osInstance, values.SysConfigName); err != nil { return values.RequeueNow, err @@ -113,13 +116,27 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re return values.RequeueNow, err } configOps, err := checkConfigVersion(osCr, osInstance, values.UpgradeConfigName) + if err != nil { + return values.RequeueNow, err + } if configOps == values.Reassign { if err = r.refreshNode(ctx, &node, osInstance, osCr.Spec.UpgradeConfigs.Version, values.UpgradeConfigName); err != nil { return values.RequeueNow, err } - return values.RequeueNow, nil + return values.Requeue, nil } + if _, ok := node.Labels[values.LabelUpgrading]; ok && + osInstance.Spec.NodeStatus == values.NodeStatusIdle.String() { + log.Info("node has upgrade label, but osInstance.spec.nodestaus idle ", + "operation:", "refesh node and wait opetaot reassgin") + if err = r.refreshNode(ctx, &node, osInstance, osCr.Spec.UpgradeConfigs.Version, + values.UpgradeConfigName); err != nil { + return values.RequeueNow, err + } + return values.Requeue, nil + } + if err := r.setConfig(ctx, osInstance, values.UpgradeConfigName); err != nil { return values.RequeueNow, err } @@ -227,6 +244,13 @@ func (r *OSReconciler) refreshNode(ctx context.Context, node *corev1.Node, osIns return err } } + if _, ok := node.Labels[values.LabelConfiguring]; ok { + delete(node.Labels, values.LabelConfiguring) + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to delete label", "node", node.Name) + return err + } + } if node.Spec.Unschedulable { // update done, uncordon the node drainer := &drain.Helper{ Ctx: ctx, @@ -261,7 +285,7 @@ func checkOsiExist(ctx context.Context, r common.ReadStatusWriter, nameSpace str Namespace: nameSpace, Name: nodeName, Labels: map[string]string{ - "upgrade.openeuler.org/osinstance-node": nodeName, + values.LabelOSinstance: nodeName, }, }, } diff --git a/cmd/proxy/controllers/os_controller_test.go b/cmd/proxy/controllers/os_controller_test.go index ff12f6458866fd09eabb2908914e64503535f2df..14b6b66672410bb172a1dc0276d1eb251c34e0bc 100644 --- a/cmd/proxy/controllers/os_controller_test.go +++ b/cmd/proxy/controllers/os_controller_test.go @@ -16,25 +16,27 @@ import ( "context" "fmt" "reflect" + "testing" "time" "github.com/agiledragon/gomonkey/v2" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - + "k8s.io/kubectl/pkg/drain" upgradev1 "openeuler.org/KubeOS/api/v1alpha1" "openeuler.org/KubeOS/pkg/agentclient" + "openeuler.org/KubeOS/pkg/common" "openeuler.org/KubeOS/pkg/values" ) var _ = Describe("OsController", func() { const ( - OSName = "test-os" - + OSName = "test-os" timeout = time.Second * 20 interval = time.Millisecond * 500 ) @@ -67,6 +69,173 @@ var _ = Describe("OsController", func() { testNamespace = existingNamespace.Name }) + AfterEach(func() { + // delete all OS CRs + osList := &upgradev1.OSList{} + err := k8sClient.List(context.Background(), osList) + Expect(err).ToNot(HaveOccurred()) + for _, os := range osList.Items { + k8sClient.Delete(context.Background(), &os) + } + osList = &upgradev1.OSList{} + Eventually(func() bool { + err = k8sClient.List(context.Background(), osList) + if err != nil || len(osList.Items) != 0 { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + }) + + Context("When we want to rollback", func() { + It("Should be able to rollback to previous version", func() { + ctx := context.Background() + + By("Creating a worker node") + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + values.LabelUpgrading: "", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v2", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + reconciler.hostName = node1Name + + By("Creating the corresponding OSInstance") + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + NodeStatus: values.NodeStatusUpgrade.String(), + SysConfigs: upgradev1.SysConfigs{}, + UpgradeConfigs: upgradev1.SysConfigs{}, + }, + Status: upgradev1.OSInstanceStatus{}, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + + // stub r.Connection.RollbackSpec() + patchRollback := gomonkey.ApplyMethodReturn(reconciler.Connection, "RollbackSpec", nil) + defer patchRollback.Reset() + patchConfigure := gomonkey.ApplyMethodReturn(reconciler.Connection, "ConfigureSpec", nil) + defer patchConfigure.Reset() + + By("Creating a OS custom resource") + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "rollback", + MaxUnavailable: 3, + OSVersion: "KubeOS v1", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) + Expect(createdOS.Spec.OpsType).Should(Equal("rollback")) + + By("Changing the nodeinfo OSImage to previous version, pretending the rollback success") + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + existingNode.Status.NodeInfo.OSImage = "KubeOS v1" + Expect(k8sClient.Status().Update(ctx, existingNode)).Should(Succeed()) + + By("Changing the OS Spec config to trigger reconcile") + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOSIns.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOSIns)).Should(Succeed()) + createdOS = &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOS)).Should(Succeed()) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + // NodeStatus changes to idle then operator can reassign configs to this node + Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(false)) + }) + }) + Context("When we have a sysconfig whose version is different from current OSInstance config version", func() { It("Should configure the node", func() { ctx := context.Background() @@ -110,6 +279,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusConfig.String(), @@ -190,7 +362,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) By("Checking the OSInstance status config version") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -245,6 +417,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusUpgrade.String(), @@ -303,7 +478,7 @@ var _ = Describe("OsController", func() { OSVersion: "KubeOS v2", FlagSafe: true, MTLS: false, - EvictPodForce: true, + EvictPodForce: false, SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, UpgradeConfigs: upgradev1.SysConfigs{ Version: "v2", @@ -330,7 +505,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) By("Checking the OSInstance status config version") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -418,7 +593,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) By("Checking the existence of new OSInstance") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns := &upgradev1.OSInstance{} Eventually(func() bool { @@ -426,6 +601,9 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + hostname, ok := createdOSIns.ObjectMeta.Labels[values.LabelOSinstance] + Expect(ok).Should(BeTrue()) + Expect(hostname).Should(Equal(node1Name)) }) }) @@ -473,6 +651,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusConfig.String(), @@ -490,7 +671,7 @@ var _ = Describe("OsController", func() { }, UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, }, - Status: upgradev1.OSInstanceStatus{}, + Status: upgradev1.OSInstanceStatus{SysConfigs: upgradev1.SysConfigs{Version: "v1"}}, } Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) @@ -557,7 +738,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) By("Checking the OSInstance status config version failed to be updated") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -567,10 +748,18 @@ var _ = Describe("OsController", func() { Expect(createdOSIns.Status.SysConfigs.Version).Should(Equal("v1")) Expect(createdOSIns.Spec.SysConfigs.Version).Should(Equal("v2")) - By("Changing the OS Spec config version to previous one") + By("Changing the OS and OSi Spec config version to previous one") OS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} Expect(k8sClient.Update(ctx, OS)).Should(Succeed()) - time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOSIns.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOSIns)).Should(Succeed()) + + time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) @@ -625,6 +814,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusUpgrade.String(), @@ -642,7 +834,6 @@ var _ = Describe("OsController", func() { }, }, }, - Status: upgradev1.OSInstanceStatus{}, } Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) @@ -713,7 +904,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) By("Checking the OSInstance status config version failed to be updated") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -732,6 +923,16 @@ var _ = Describe("OsController", func() { err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) return err == nil }, timeout, interval).Should(BeTrue()) + + osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOSIns.Spec.UpgradeConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOSIns)).Should(Succeed()) + // NodeStatus changes to idle then operator can reassign configs to this node Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) existingNode = &v1.Node{} @@ -749,37 +950,7 @@ var _ = Describe("OsController", func() { It("Should be able to rollback to previous config version to jump out of error state", func() { ctx := context.Background() - By("Creating a worker node") node1Name = "test-node-" + uuid.New().String() - node1 := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: node1Name, - Namespace: testNamespace, - Labels: map[string]string{ - "beta.kubernetes.io/os": "linux", - values.LabelUpgrading: "", - }, - }, - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - Status: v1.NodeStatus{ - NodeInfo: v1.NodeSystemInfo{ - OSImage: "KubeOS v2", - }, - }, - } - err := k8sClient.Create(ctx, node1) - Expect(err).ToNot(HaveOccurred()) - existingNode := &v1.Node{} - Eventually(func() bool { - err := k8sClient.Get(context.Background(), - types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) - return err == nil - }, timeout, interval).Should(BeTrue()) - reconciler.hostName = node1Name - By("Creating the corresponding OSInstance") OSIns := &upgradev1.OSInstance{ TypeMeta: metav1.TypeMeta{ @@ -789,6 +960,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusUpgrade.String(), @@ -844,6 +1018,37 @@ var _ = Describe("OsController", func() { createdOSIns.Status.SysConfigs.Version = "v1" Expect(k8sClient.Status().Update(ctx, createdOSIns)).Should(Succeed()) Expect(createdOSIns.Status.UpgradeConfigs.Version).Should(Equal("v2")) + Expect(createdOSIns.Status.SysConfigs.Version).Should(Equal("v1")) + + By("Creating a worker node") + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + values.LabelUpgrading: "", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v2", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + reconciler.hostName = node1Name // stub r.Connection.ConfigureSpec() patchConfigure := gomonkey.ApplyMethod(reflect.TypeOf(reconciler.Connection), @@ -906,8 +1111,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) By("Checking the OSInstance status config version failed to be updated") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished - osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) @@ -917,8 +1121,21 @@ var _ = Describe("OsController", func() { Expect(createdOSIns.Spec.SysConfigs.Version).Should(Equal("v2")) By("Changing the OS Spec config version to previous one") - OS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} - Expect(k8sClient.Update(ctx, OS)).Should(Succeed()) + createdOS = &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOS)).Should(Succeed()) + getOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, getOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + getOSIns.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, getOSIns)).Should(Succeed()) + time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -938,4 +1155,274 @@ var _ = Describe("OsController", func() { Expect(ok).Should(Equal(false)) }) }) + + Context("When node has upgrade label but osinstance.spec.nodestatus is idle", func() { + It("Should be able to refresh node and wait operator reassgin upgrade", func() { + ctx := context.Background() + By("Creating a worker node") + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + values.LabelUpgrading: "", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v2", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + reconciler.hostName = node1Name + + By("Creating the corresponding OSInstance") + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + NodeStatus: values.NodeStatusIdle.String(), + }, + Status: upgradev1.OSInstanceStatus{}, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + By("Creating a OS custom resource") + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "upgrade", + MaxUnavailable: 3, + OSVersion: "KubeOS v2", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "c"}, + {Key: "key2", Value: "d"}, + }, + }, + }, + }, + UpgradeConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) + + time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(false)) + }) + }) }) + +func Test_evictNode(t *testing.T) { + type args struct { + drainer *drain.Helper + node *corev1.Node + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "node unschedulable", + args: args{ + drainer: &drain.Helper{}, + node: &corev1.Node{Spec: v1.NodeSpec{Unschedulable: true}}, + }, + wantErr: false, + }, + { + name: "runCordonError1", + args: args{ + drainer: &drain.Helper{}, + node: &corev1.Node{}, + }, + wantErr: true, + }, + { + name: "runNodeDrainError", + args: args{ + drainer: &drain.Helper{}, + node: &corev1.Node{}, + }, + wantErr: true, + }, + { + name: "runUncordonError2", + args: args{ + drainer: &drain.Helper{}, + node: &corev1.Node{}, + }, + wantErr: true, + }, + } + patchRunCordon := gomonkey.ApplyFuncSeq(drain.RunCordonOrUncordon, []gomonkey.OutputCell{ + {Values: gomonkey.Params{fmt.Errorf("cordon error")}}, + {Values: gomonkey.Params{nil}}, + {Values: gomonkey.Params{fmt.Errorf("cordon error")}}, + {Values: gomonkey.Params{nil}}, + {Values: gomonkey.Params{nil}}, + }) + defer patchRunCordon.Reset() + patchRunNodeDrain := gomonkey.ApplyFuncSeq(drain.RunNodeDrain, []gomonkey.OutputCell{ + {Values: gomonkey.Params{fmt.Errorf("node drain error")}}, + {Values: gomonkey.Params{fmt.Errorf("node drain error")}}, + }) + defer patchRunNodeDrain.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := evictNode(tt.args.drainer, tt.args.node); (err != nil) != tt.wantErr { + t.Errorf("evictNode() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_updateConfigStatus(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + osInstance *upgradev1.OSInstance + configType string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "invalid config type", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + osInstance: &upgradev1.OSInstance{}, + configType: "invalid", + }, + wantErr: true, + }, + } + patchUpdate := gomonkey.ApplyMethodReturn(&OSReconciler{}, "Update", fmt.Errorf("update error")) + patchStatus := gomonkey.ApplyMethodReturn(&OSReconciler{}, "Status", &OSReconciler{}) + defer patchUpdate.Reset() + defer patchStatus.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := updateConfigStatus(tt.args.ctx, tt.args.r, tt.args.osInstance, tt.args.configType); (err != nil) != tt.wantErr { + t.Errorf("updateConfigStatus() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_getOSAndNodeStatus(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + name types.NamespacedName + hostName string + } + tests := []struct { + name string + args args + wantOS upgradev1.OS + wantNode corev1.Node + }{ + { + name: "get node error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + name: types.NamespacedName{}, + hostName: "test-node", + }, + wantOS: upgradev1.OS{}, + wantNode: corev1.Node{}, + }, + } + patchGet := gomonkey.ApplyMethodSeq(&OSReconciler{}, "Get", []gomonkey.OutputCell{ + {Values: gomonkey.Params{nil}}, + {Values: gomonkey.Params{fmt.Errorf("get node error")}}, + }) + defer patchGet.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotOS, gotNode := getOSAndNodeStatus(tt.args.ctx, tt.args.r, tt.args.name, tt.args.hostName) + if !reflect.DeepEqual(gotOS, tt.wantOS) { + t.Errorf("getOSAndNodeStatus() gotOS = %v, want %v", gotOS, tt.wantOS) + } + if !reflect.DeepEqual(gotNode, tt.wantNode) { + t.Errorf("getOSAndNodeStatus() gotNode = %v, want %v", gotNode, tt.wantNode) + } + }) + } +} diff --git a/cmd/proxy/controllers/suite_test.go b/cmd/proxy/controllers/suite_test.go index a52c18f5e44ed4f71283cc57ff82a288ea19d676..00eebbf4a6a652cd897046e7e5a96bb9f7ad3d8f 100644 --- a/cmd/proxy/controllers/suite_test.go +++ b/cmd/proxy/controllers/suite_test.go @@ -53,7 +53,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "docs", "example", "config", "crd")}, ErrorIfCRDPathMissing: true, } diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index ce1f58dfed716c7f8f20df661b130c2cae7f733b..3a537d90375a34a700b66136e11e844ef888a15e 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -44,10 +44,14 @@ func init() { } func main() { - mgr := common.NewControllerManager(setupLog, scheme) + var err error + mgr, err := common.NewControllerManager(setupLog, scheme) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } reconciler := controllers.NewOSReconciler(mgr) - var err error if reconciler.Connection, err = agentclient.New("unix://" + filepath.Join(server.SockDir, server.SockName)); err != nil { setupLog.Error(err, "Error running proxy") } diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index 3bb1333d6973ce2f6a0513ffb5ff76b03bae1ceb..acc70fc08c676b2c58ac836248df148417b9218a 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -54,6 +54,8 @@ spec: type: integer mtls: type: boolean + nodeselector: + type: string opstype: type: string osversion: diff --git a/docs/example/config/samples/upgrade_v1alpha1_os.yaml b/docs/example/config/samples/upgrade_v1alpha1_os.yaml index b33a30b2602eea0104479bf104607adba799b240..ca3da1b2c9a52651f458f33bbafca5435ae2f970 100644 --- a/docs/example/config/samples/upgrade_v1alpha1_os.yaml +++ b/docs/example/config/samples/upgrade_v1alpha1_os.yaml @@ -13,6 +13,7 @@ spec: checksum: image digests flagSafe: false mtls: false + nodeselector: edit.nodes.label sysconfigs: version: edit.sysconfig.version configs: @@ -35,4 +36,4 @@ spec: contents: - key: kernel param key4 value: kernel param value4 - operation: delete \ No newline at end of file + operation: delete diff --git a/docs/quick-start.md b/docs/quick-start.md index 0d4dc4bf98fd9899ef7593f9e8b51d03f6652a31..748e0819069b995d17f7830f51bb22a8a876b61a 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -176,6 +176,7 @@ | clientcert | string | https双向认证时使用的客户端证书文件 | 仅在使用https双向认证时有效|mtls为true时必选 | | clientkey | string | https双向认证时使用的客户端公钥 | 仅在使用https双向认证时有效|mtls为true时必选 | | evictpodforce | bool | 用于表示升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | + | nodeselector | string | 需要进行升级/配置/回滚操作的节点label | 用于只对具有某些特定label的节点而不是集群所有worker节点进行运维的场景，需要进行运维操作的节点需要包含key为upgrade.openeuler.org/node-selector的label，nodeselector为该label的value值，此参数不配置时，或者配置为""时默认对所有节点进行操作| 可选 | | sysconfigs | / | 需要进行配置的参数值 | 在配置或者升级或者回退机器时有效，在升级或者回退操作之后即机器重启之后起效，详细字段说明请见```配置（Settings）指导```| 可选 | | upgradeconfigs | / | 需要升级前进行的配置的参数值 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | @@ -247,7 +248,7 @@ mtls: true ``` - * 升级并且进行配置的示例如下， + * 升级并且进行配置的示例如下 * 以节点容器引擎为containerd为例，升级方式对配置无影响，upgradeconfigs在升级前起效，sysconfigs在升级后起效，配置参数说明请见```配置(Settings)指导``` * 升级并且配置时opstype字段需为upgrade * upgradeconfig为升级之前执行的配置，sysconfigs为升级机器重启后执行的配置，用户可按需进行配置 @@ -292,7 +293,37 @@ - key: kernel param key4 value: kernel param value4 ``` + * 只升级部分节点示例如下 + * 以节点容器引擎为containerd为例，升级方式对节点筛选无影响 + * 需要进行升级的节点需包含key为upgrade.openeuler.org/node-selector的label，nodeselector的值为该label的value，即假定nodeselector值为kubeos，则只对包含upgrade.openeuler.org/node-selector=kubeos的label的worker节点进行升级 + * nodeselector对配置和回滚同样有效 + * 节点添加label和label修改命令示例如下： + ``` shell + # 为节点kubeos-node1增加label + kubectl label nodes kubeos-node1 upgrade.openeuler.org/node-selector=kubeos-v1 + # 修改节点kubeos-node1的label + kubectl label --overwrite nodes kubeos-node2 upgrade.openeuler.org/node-selector=kubeos-v2 + ``` + * yaml示例如下： + ```yaml + apiVersion: upgrade.openeuler.org/v1alpha1 + kind: OS + metadata: + name: os-sample + spec: + imagetype: containerd + opstype: upgrade + osversion: edit.os.version + maxunavailable: edit.node.upgrade.number + containerimage: container image like repository/name:tag + evictpodforce: true/false + imageurl: "" + checksum: container image digests + flagSafe: false + mtls: true + nodeselector: edit.node.label.key + ``` * 查看未升级的节点的 OS 版本 ```shell @@ -366,20 +397,20 @@ * 查看配置之前的节点的配置的版本和节点状态（NODESTATUS状态为idle） ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' ``` * 执行命令，在集群中部署cr实例后，节点会根据配置的参数信息进行配置，再次查看节点状态(NODESTATUS变成config) ```shell kubectl apply -f upgrade_v1alpha1_os.yaml - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' ``` * 再次查看节点的配置的版本确认节点是否配置完成(NODESTATUS恢复为idle) ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' ``` * 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 @@ -605,7 +636,7 @@ hostshell #### kernel Settings -* kenerl.sysctl: 设置内核参数，key/value 表示内核参数的 key/value, 示例如下: +* kenerl.sysctl: 临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）, 示例如下: ```yaml configs: @@ -618,8 +649,7 @@ hostshell operation: delete ``` -* kernel.sysctl.persist: 设置持久化内核参数，key/value 表示内核参数的 key/value, configpath为配置修改/新建的文件路径，如不指定configpath默认修改/etc/sysctl.conf - +* kernel.sysctl.persist: 设置持久化内核参数，key/value表示内核参数的key/value，key与value均不能为空且key不能包含“=”， configpath为配置文件路径，支持新建（需保证父目录存在），如不指定configpath默认修改/etc/sysctl.conf，示例如下： ```yaml configs: - model: kernel.systcl.persist @@ -637,13 +667,14 @@ hostshell * grub.cmdline: 设置grub.cfg文件中的内核引导参数，该行参数在grub.cfg文件中类似如下示例： ```shell - linux /boot/vmlinuz root=UUID=5b1aaf5d-5b25-4e4b-a0d3-3d4c8d2e6a6e ro consoleblank=600 console=tty0 console=ttyS0,115200n8 selinux=1 panic=3 + linux /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 ``` - key/value 表示如上示例中内核引导参数的 key=value。 - **注意：** 当该参数有多个等号，如root=UUID=some-uuid时，配置时的key为第一个等号前的所有字符，value为第一个等号后的所有字符。 - 配置方法示例如下： - +* KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置： + * grub.cmdline.current：对当前分区的启动项参数进行配置。 + * grub.cmdline.next：对下一分区的启动项参数进行配置。 +* 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。 +* grub.cmdline.current/next支持“key=value”（value不能为空），也支持单key。若value中有“=”，例如“root=UUID=some-uuid”，key应设置为第一个“=”前的所有字符，value为第一个“=”后的所有字符。 配置方法示例如下： ```yaml configs: - model: grub.cmdline diff --git a/pkg/common/common.go b/pkg/common/common.go index 4653a61984a419919b9f816d6d6c4d9245c23347..e888179fe2c3cad7c37e9a979a01c9a169e70699 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -15,7 +15,6 @@ package common import ( "flag" - "os" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" @@ -33,7 +32,7 @@ type ReadStatusWriter interface { } // NewControllerManager configure and return a manager -func NewControllerManager(setupLog logr.Logger, scheme *runtime.Scheme) manager.Manager { +func NewControllerManager(setupLog logr.Logger, scheme *runtime.Scheme) (manager.Manager, error) { opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -46,7 +45,7 @@ func NewControllerManager(setupLog logr.Logger, scheme *runtime.Scheme) manager. }) if err != nil { setupLog.Error(err, "unable to start manager") - os.Exit(1) + return nil, err } - return mgr + return mgr, nil } diff --git a/pkg/values/values.go b/pkg/values/values.go index 30261bd35749268d7f7646b3e0572a4bf22c926b..e2c035407e80f05bde84a4e6e59fad277cb671b4 100644 --- a/pkg/values/values.go +++ b/pkg/values/values.go @@ -23,8 +23,14 @@ const ( // LabelUpgrading is the key of the upgrading label for nodes LabelUpgrading = "upgrade.openeuler.org/upgrading" // LabelMaster is the key of the master-node label for nodes - LabelMaster = "node-role.kubernetes.io/control-plane" - defaultPeriod = 15 * time.Second + LabelMaster = "node-role.kubernetes.io/control-plane" + // LabelOSinstance is used to select the osinstance with the nodeName by label + LabelOSinstance = "upgrade.openeuler.org/osinstance-node" + // LabelNodeSelector is used to filter the nodes that need to be upgraded or configured. + LabelNodeSelector = "upgrade.openeuler.org/node-selector" + // LabelConfiguring is the key of the configuring label for nodes + LabelConfiguring = "upgrade.openeuler.org/configuring" + defaultPeriod = 15 * time.Second // OsiStatusName is param name of nodeStatus in osInstance OsiStatusName = "nodestatus" // UpgradeConfigName is param name of UpgradeConfig diff --git a/scripts/admin-container/set-ssh-pub-key.service b/scripts/admin-container/set-ssh-pub-key.service index cf214069fe8e5c1568abe5b34795b5b43924509f..84dd12d928d93d8c9a4c9fbe6c22973b3804715e 100644 --- a/scripts/admin-container/set-ssh-pub-key.service +++ b/scripts/admin-container/set-ssh-pub-key.service @@ -12,4 +12,4 @@ Description="set ssh authorized keys according to the secret which is set by user" [Service] -ExecStart="/usr/local/bin/set-ssh-pub-key.sh" \ No newline at end of file +ExecStart="/usr/local/bin/set-ssh-pub-key.sh" diff --git a/scripts/admin-container/set-ssh-pub-key.sh b/scripts/admin-container/set-ssh-pub-key.sh index aa706c2d1893c71a93074236a21412eb8f7aaa07..e91a15de3500ae0e8b54b9165f122d2f8e077301 100755 --- a/scripts/admin-container/set-ssh-pub-key.sh +++ b/scripts/admin-container/set-ssh-pub-key.sh @@ -23,4 +23,4 @@ if [ ! -f "$authorized_file" ]; then chmod 600 "$authorized_file" fi -echo "$ssh_pub" >> "$authorized_file" \ No newline at end of file +echo "$ssh_pub" >> "$authorized_file"