From 8ac73f031de82394ccb795263d20be2d954d201f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B8=85?= Date: Tue, 25 Apr 2023 16:43:52 +0800 Subject: [PATCH 01/27] KubeOS: update test --- cmd/agent/server/server_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/agent/server/server_test.go b/cmd/agent/server/server_test.go index f7407ff1..272c9f98 100644 --- a/cmd/agent/server/server_test.go +++ b/cmd/agent/server/server_test.go @@ -179,9 +179,9 @@ func TestServerupdate(t *testing.T) { }}, wantErr: true}, {name: "errordocker", args: args{&pb.UpdateRequest{ - DockerImage: "", - ImageType: "docker", - Certs: &pb.CertsInfo{}, + ContainerImage: "", + ImageType: "docker", + Certs: &pb.CertsInfo{}, }}, wantErr: true}, } -- Gitee From 790d53dc581874575aef1777122856a59bcdbf8b Mon Sep 17 00:00:00 2001 From: liyuanr Date: Thu, 3 Aug 2023 22:18:23 +0800 Subject: [PATCH 02/27] KubeOS:add unit tests of containerd and docker,modify code for cleancode Add unit tests of using containerd or docker to upgrade. Modify code for cleancode and fix issue of ctr images pull Signed-off-by: liyuanr --- cmd/agent/server/containerd_image.go | 8 +- cmd/agent/server/containerd_image_test.go | 139 ++++++++++++++++++++++ cmd/agent/server/docker_image.go | 2 +- cmd/agent/server/docker_image_test.go | 88 ++++++++++++-- cmd/agent/server/utils.go | 102 ++++++++-------- 5 files changed, 272 insertions(+), 67 deletions(-) create mode 100644 cmd/agent/server/containerd_image_test.go diff --git a/cmd/agent/server/containerd_image.go b/cmd/agent/server/containerd_image.go index f180fb54..fd612745 100644 --- a/cmd/agent/server/containerd_image.go +++ b/cmd/agent/server/containerd_image.go @@ -23,9 +23,7 @@ import ( pb "openeuler.org/KubeOS/cmd/agent/api" ) -var ( - defaultNamespace = "k8s.io" -) +const defaultNamespace = "k8s.io" type conImageHandler struct{} @@ -56,7 +54,7 @@ func (c conImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath prep } } else { containerdCommand = "ctr" - if err := runCommand("ctr", "-n", defaultNamespace, "images", "pull", "--host-dir", + if err := runCommand("ctr", "-n", defaultNamespace, "images", "pull", "--hosts-dir", "/etc/containerd/certs.d", imageName); err != nil { return "", err } @@ -76,7 +74,7 @@ func (c conImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath prep return "", err } defer checkAndCleanMount(mountPath) - if err := copyFile(neededPath.tarPath, mountPath+"/"+rootfsArchive); err != nil { + if err := copyFile(neededPath.tarPath, mountPath+"/"+neededPath.rootfsFile); err != nil { return "", err } return "", nil diff --git a/cmd/agent/server/containerd_image_test.go b/cmd/agent/server/containerd_image_test.go new file mode 100644 index 00000000..d7133c37 --- /dev/null +++ b/cmd/agent/server/containerd_image_test.go @@ -0,0 +1,139 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package server implements server of os-agent and listener of os-agent server. The server uses gRPC interface. +package server + +import ( + "os" + "testing" + + "github.com/agiledragon/gomonkey/v2" + pb "openeuler.org/KubeOS/cmd/agent/api" +) + +func Test_conImageHandler_downloadImage(t *testing.T) { + type args struct { + req *pb.UpdateRequest + } + tests := []struct { + name string + c conImageHandler + args args + want string + wantErr bool + }{ + + { + name: "pullImageError", + c: conImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "testError"}, + }, + want: "", + wantErr: true, + }, + { + name: "checkSumError", + c: conImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "docker.io/library/hello-world:latest"}, + }, + want: "", + wantErr: true, + }, + { + name: "normal", + c: conImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ + ContainerImage: "docker.io/library/hello-world:latest", + }, + }, + want: "update-test1/upadte.img", + wantErr: false, + }, + } + patchPrepareEnv := gomonkey.ApplyFunc(prepareEnv, func() (preparePath, error) { + return preparePath{updatePath: "update-test1/", + mountPath: "update-test1/mountPath", + tarPath: "update-test1/mountPath/hello", + imagePath: "update-test1/upadte.img", + rootfsFile: "hello"}, nil + }) + defer patchPrepareEnv.Reset() + patchCreateOSImage := gomonkey.ApplyFunc(createOSImage, func(neededPath preparePath) (string, error) { + return "update-test1/upadte.img", nil + }) + defer patchCreateOSImage.Reset() + + if err := os.MkdirAll("update-test1/mountPath", os.ModePerm); err != nil { + t.Errorf("create test dir error = %v", err) + return + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := conImageHandler{} + if tt.name == "normal" { + imageDigests, err := getOCIImageDigest("crictl", "docker.io/library/hello-world:latest") + if err != nil { + t.Errorf("conImageHandler.getRootfsArchive() get oci image digests error = %v", err) + } + tt.args.req.CheckSum = imageDigests + } + got, err := c.downloadImage(tt.args.req) + if (err != nil) != tt.wantErr { + t.Errorf("conImageHandler.downloadImage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("conImageHandler.downloadImage() = %v, want %v", got, tt.want) + } + }) + } + defer func() { + if err := runCommand("crictl", "rmi", "docker.io/library/hello-world:latest"); err != nil { + t.Errorf("remove kubeos-temp container error = %v", err) + } + if err := os.RemoveAll("update-test1"); err != nil { + t.Errorf("remove update-test error = %v", err) + } + }() +} + +func Test_copyFile(t *testing.T) { + type args struct { + dstFileName string + srcFileName string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "srcFileNotExist", + args: args{ + dstFileName: "bbb.txt", + srcFileName: "aaa.txt", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := copyFile(tt.args.dstFileName, tt.args.srcFileName); (err != nil) != tt.wantErr { + t.Errorf("copyFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/agent/server/docker_image.go b/cmd/agent/server/docker_image.go index 23e596b3..0b6ee35b 100644 --- a/cmd/agent/server/docker_image.go +++ b/cmd/agent/server/docker_image.go @@ -61,7 +61,7 @@ func (d dockerImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath p if err != nil { return "", err } - if err := runCommand("docker", "cp", containerId+":/"+rootfsArchive, neededPath.updatePath); err != nil { + if err := runCommand("docker", "cp", containerId+":/"+neededPath.rootfsFile, neededPath.updatePath); err != nil { return "", err } defer func() { diff --git a/cmd/agent/server/docker_image_test.go b/cmd/agent/server/docker_image_test.go index 99879393..2dbf3370 100644 --- a/cmd/agent/server/docker_image_test.go +++ b/cmd/agent/server/docker_image_test.go @@ -17,38 +17,102 @@ import ( "os" "testing" + "github.com/agiledragon/gomonkey/v2" pb "openeuler.org/KubeOS/cmd/agent/api" ) -func TestpullOSImage(t *testing.T) { +func Test_dockerImageHandler_downloadImage(t *testing.T) { type args struct { req *pb.UpdateRequest } - os.Mkdir("/persist", os.ModePerm) tests := []struct { name string + d dockerImageHandler args args want string wantErr bool }{ - {name: "pull image error", args: args{req: &pb.UpdateRequest{ - DockerImage: "test", - }}, want: "", wantErr: true}, - {name: "normal", args: args{req: &pb.UpdateRequest{ - DockerImage: "centos", - }}, want: "/persist/update.img", wantErr: false}, + { + name: "pullImageError", + d: dockerImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "testError"}, + }, + want: "", + wantErr: true, + }, + + { + name: "checkSumError", + d: dockerImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "hello-world", CheckSum: "aaaaaa"}, + }, + want: "", + wantErr: true, + }, + + { + name: "normal", + d: dockerImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "hello-world"}, + }, + want: "update-test/upadte.img", + wantErr: false, + }, + } + patchPrepareEnv := gomonkey.ApplyFunc(prepareEnv, func() (preparePath, error) { + return preparePath{updatePath: "update-test/", + mountPath: "update-test/mountPath", + tarPath: "update-test/mountPath/hello", + imagePath: "update-test/upadte.img", + rootfsFile: "hello"}, nil + }) + defer patchPrepareEnv.Reset() + + patchCreateOSImage := gomonkey.ApplyFunc(createOSImage, func(neededPath preparePath) (string, error) { + return "update-test/upadte.img", nil + }) + defer patchCreateOSImage.Reset() + + if err := os.MkdirAll("update-test/mountPath", os.ModePerm); err != nil { + t.Errorf("create test dir error = %v", err) + return } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := pullOSImage(tt.args.req) + if tt.name == "normal" { + _, err := runCommandWithOut("docker", "create", "--name", "kubeos-temp", "hello-world") + if err != nil { + t.Errorf("Test_dockerImageHandler_getRootfsArchive create container error = %v", err) + return + } + imageDigests, err := getOCIImageDigest("docker", "hello-world") + + if err != nil { + t.Errorf("Test_dockerImageHandler_getRootfsArchive get oci image digests error = %v", err) + } + tt.args.req.CheckSum = imageDigests + } + d := dockerImageHandler{} + got, err := d.downloadImage(tt.args.req) if (err != nil) != tt.wantErr { - t.Errorf("pullOSImage() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("dockerImageHandler.downloadImage() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("pullOSImage() = %v, want %v", got, tt.want) + t.Errorf("dockerImageHandler.downloadImage() = %v, want %v", got, tt.want) } }) } - defer os.RemoveAll("/persist") + defer func() { + if err := runCommand("docker", "rmi", "hello-world"); err != nil { + t.Errorf("remove kubeos-temp container error = %v", err) + } + if err := os.RemoveAll("update-test"); err != nil { + t.Errorf("remove update-test error = %v", err) + } + }() } diff --git a/cmd/agent/server/utils.go b/cmd/agent/server/utils.go index c8a72c3e..7134d74f 100644 --- a/cmd/agent/server/utils.go +++ b/cmd/agent/server/utils.go @@ -31,10 +31,7 @@ import ( const ( needGBSize = 3 // the max size of update files needed // KB is 1024 B - KB = 1024 -) - -var ( + KB = 1024 rootfsArchive = "os.tar" updateDir = "KubeOS-Update" mountDir = "kubeos-update" @@ -51,6 +48,7 @@ type preparePath struct { mountPath string tarPath string imagePath string + rootfsFile string } func runCommand(name string, args ...string) error { @@ -192,9 +190,10 @@ func prepareEnv() (preparePath, error) { if err := checkDiskSize(needGBSize, PersistDir); err != nil { return preparePath{}, err } + rootfsFile := rootfsArchive updatePath := splicePath(PersistDir, updateDir) mountPath := splicePath(updatePath, mountDir) - tarPath := splicePath(updatePath, rootfsArchive) + tarPath := splicePath(updatePath, rootfsFile) imagePath := splicePath(PersistDir, osImageName) if err := cleanSpace(updatePath, mountPath, imagePath); err != nil { @@ -208,6 +207,7 @@ func prepareEnv() (preparePath, error) { mountPath: mountPath, tarPath: tarPath, imagePath: imagePath, + rootfsFile: rootfsFile, } return upgradePath, nil } @@ -284,50 +284,9 @@ func checkFileExist(path string) (bool, error) { } func checkOCIImageDigestMatch(containerRuntime string, imageName string, checkSum string) error { - var cmdOutput string - var err error - switch containerRuntime { - case "crictl": - cmdOutput, err = runCommandWithOut("crictl", "inspecti", "--output", "go-template", - "--template", "{{.status.repoDigests}}", imageName) - if err != nil { - return err - } - case "docker": - cmdOutput, err = runCommandWithOut("docker", "inspect", "--format", "{{.RepoDigests}}", imageName) - if err != nil { - return err - } - case "ctr": - cmdOutput, err = runCommandWithOut("ctr", "-n", "k8s.io", "images", "ls", "name=="+imageName) - if err != nil { - return err - } - // after Fields, we get slice like [REF TYPE DIGEST SIZE PLATFORMS LABELS x x x x x x] - // the digest is the position 8 element - imageDigest := strings.Split(strings.Fields(cmdOutput)[8], ":")[1] - if imageDigest != checkSum { - logrus.Errorln("checkSumFailed ", imageDigest, " mismatch to ", checkSum) - return fmt.Errorf("checkSumFailed %s mismatch to %s", imageDigest, checkSum) - } - return nil - default: - logrus.Errorln("containerRuntime ", containerRuntime, " cannot be recognized") - return fmt.Errorf("containerRuntime %s cannot be recognized", containerRuntime) - } - // cmdOutput format is as follows: - // [imageRepository/imageName:imageTag@sha256:digests] - // parse the output and get digest - var imageDigests string - outArray := strings.Split(cmdOutput, "@") - if strings.HasPrefix(outArray[len(outArray)-1], "sha256") { - pasredArray := strings.Split(strings.TrimSuffix(outArray[len(outArray)-1], "]"), ":") - // 2 is the expected length of the array after dividing "imageName:imageTag@sha256:digests" based on ':' - rightLen := 2 - if len(pasredArray) == rightLen { - digestIndex := 1 // 1 is the index of digest data in pasredArray - imageDigests = pasredArray[digestIndex] - } + imageDigests, err := getOCIImageDigest(containerRuntime, imageName) + if err != nil { + return err } if imageDigests == "" { logrus.Errorln("error when get ", imageName, " digests") @@ -367,3 +326,48 @@ func isValidImageName(image string) error { } return nil } + +func getOCIImageDigest(containerRuntime string, imageName string) (string, error) { + var cmdOutput string + var err error + var imageDigests string + switch containerRuntime { + case "crictl": + cmdOutput, err = runCommandWithOut("crictl", "inspecti", "--output", "go-template", + "--template", "{{.status.repoDigests}}", imageName) + if err != nil { + return "", err + } + case "docker": + cmdOutput, err = runCommandWithOut("docker", "inspect", "--format", "{{.RepoDigests}}", imageName) + if err != nil { + return "", err + } + case "ctr": + cmdOutput, err = runCommandWithOut("ctr", "-n", "k8s.io", "images", "ls", "name=="+imageName) + if err != nil { + return "", err + } + // after Fields, we get slice like [REF TYPE DIGEST SIZE PLATFORMS LABELS x x x x x x] + // the digest is the position 8 element + imageDigest := strings.Split(strings.Fields(cmdOutput)[8], ":")[1] + return imageDigest, nil + default: + logrus.Errorln("containerRuntime ", containerRuntime, " cannot be recognized") + return "", fmt.Errorf("containerRuntime %s cannot be recognized", containerRuntime) + } + // cmdOutput format is as follows: + // [imageRepository/imageName:imageTag@sha256:digests] + // parse the output and get digest + outArray := strings.Split(cmdOutput, "@") + if strings.HasPrefix(outArray[len(outArray)-1], "sha256") { + pasredArray := strings.Split(strings.TrimSuffix(outArray[len(outArray)-1], "]"), ":") + // 2 is the expected length of the array after dividing "imageName:imageTag@sha256:digests" based on ':' + rightLen := 2 + if len(pasredArray) == rightLen { + digestIndex := 1 // 1 is the index of digest data in pasredArray + imageDigests = pasredArray[digestIndex] + } + } + return imageDigests, nil +} -- Gitee From 64e36143591c014609fc517bcd6ee0f3ae1087fc Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 3 Aug 2023 11:01:22 +0800 Subject: [PATCH 03/27] KubeOS: refactor assignUpgrade function the assignUpgrade function is refactored for maintainability replace "osinstance-node" to variable Signed-off-by: Yuhang Wei --- cmd/operator/controllers/os_controller.go | 73 ++++++----- .../controllers/os_controller_test.go | 121 +++++++++++++++++- cmd/proxy/controllers/os_controller.go | 2 +- cmd/proxy/controllers/os_controller_test.go | 9 ++ pkg/values/values.go | 6 +- 5 files changed, 176 insertions(+), 35 deletions(-) diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index f86a0b2c..620739be 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -109,7 +109,7 @@ func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLimitingInterface) { ctx := context.Background() hostname := e.Object.GetName() - labelSelector := labels.SelectorFromSet(labels.Set{"upgrade.openeuler.org/osinstance-node": hostname}) + labelSelector := labels.SelectorFromSet(labels.Set{values.LabelOSinstance: hostname}) osInstanceList := &upgradev1.OSInstanceList{} if err := r.List(ctx, osInstanceList, client.MatchingLabelsSelector{Selector: labelSelector}); err != nil { log.Error(err, "unable to list osInstances") @@ -157,12 +157,23 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. return false, err } - nodes, err := getNodes(ctx, r, limit+1, *requirement, *reqMaster) // one more to see if all node updated + nodes, err := getNodes(ctx, r, limit+1, *requirement, *reqMaster) // one more to see if all nodes updated if err != nil { return false, err } - var count = 0 + // Upgrade OS for selected nodes + count, err := upgradeNodes(ctx, r, &os, nodes, limit) + if err != nil { + return false, err + } + + return count >= limit, nil +} + +func upgradeNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) { + var count int for _, node := range nodes { if count >= limit { break @@ -170,43 +181,45 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. osVersionNode := node.Status.NodeInfo.OSImage if os.Spec.OSVersion != osVersionNode { var osInstance upgradev1.OSInstance - if err = r.Get(ctx, types.NamespacedName{Namespace: nameSpace, Name: node.Name}, &osInstance); err != nil { + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { if err = client.IgnoreNotFound(err); err != nil { log.Error(err, "failed to get osInstance "+node.Name) - return false, err + return count, err } continue } + updateNodeAndOSins(ctx, r, os, &node, &osInstance) count++ - node.Labels[values.LabelUpgrading] = "" - expUpVersion := os.Spec.UpgradeConfigs.Version - osiUpVersion := osInstance.Spec.UpgradeConfigs.Version - if osiUpVersion != expUpVersion { - osInstance.Spec.UpgradeConfigs = os.Spec.UpgradeConfigs - } - expSysVersion := os.Spec.SysConfigs.Version - osiSysVersion := osInstance.Spec.SysConfigs.Version - if osiSysVersion != expSysVersion { - osInstance.Spec.SysConfigs = os.Spec.SysConfigs - for i, config := range osInstance.Spec.SysConfigs.Configs { - if config.Model == "grub.cmdline.current" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" - } - if config.Model == "grub.cmdline.next" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" - } - } - } - osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() - if err = r.Update(ctx, &osInstance); err != nil { - log.Error(err, "unable to update", "osInstance", osInstance.Name) + } + } + return count, nil +} + +func updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) { + if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { + osInstance.Spec.UpgradeConfigs = os.Spec.UpgradeConfigs + } + if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { + osInstance.Spec.SysConfigs = os.Spec.SysConfigs + // exchange "grub.cmdline.current" and "grub.cmdline.next" + for i, config := range osInstance.Spec.SysConfigs.Configs { + if config.Model == "grub.cmdline.current" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" } - if err = r.Update(ctx, &node); err != nil { - log.Error(err, "unable to label", "node", node.Name) + if config.Model == "grub.cmdline.next" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" } } } - return count >= limit, nil + osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + } + node.Labels[values.LabelUpgrading] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + } } func assignConfig(ctx context.Context, r common.ReadStatusWriter, sysConfigs upgradev1.SysConfigs, diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index 30a773ca..a3910057 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -134,6 +134,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ SysConfigs: upgradev1.SysConfigs{ @@ -189,7 +192,7 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -301,7 +304,7 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished configedOSIns := &upgradev1.OSInstance{} Eventually(func() bool { err := k8sClient.Get(ctx, osInsCRLookupKey, configedOSIns) @@ -388,4 +391,118 @@ var _ = Describe("OsController", func() { }) }) + Context("When we want to upgrade and do sysconfigs which contain grub.cmd.current and .next", func() { + It("Should exchange .current and .next", func() { + ctx := context.Background() + + // create Node + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + + // create OS CR + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "upgrade", + MaxUnavailable: 3, + OSVersion: "KubeOS v2", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + {Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}}, + {Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}}, + }, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + // Check that the corresponding OS CR has been created + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.Spec.SysConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + Expect(createdOSIns.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) + }) + }) }) diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index a17afacf..b0b17e73 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -261,7 +261,7 @@ func checkOsiExist(ctx context.Context, r common.ReadStatusWriter, nameSpace str Namespace: nameSpace, Name: nodeName, Labels: map[string]string{ - "upgrade.openeuler.org/osinstance-node": nodeName, + values.LabelOSinstance: nodeName, }, }, } diff --git a/cmd/proxy/controllers/os_controller_test.go b/cmd/proxy/controllers/os_controller_test.go index ff12f645..e6cd5b7d 100644 --- a/cmd/proxy/controllers/os_controller_test.go +++ b/cmd/proxy/controllers/os_controller_test.go @@ -110,6 +110,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusConfig.String(), @@ -245,6 +248,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusUpgrade.String(), @@ -426,6 +432,9 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + hostname, ok := createdOSIns.ObjectMeta.Labels[values.LabelOSinstance] + Expect(ok).Should(BeTrue()) + Expect(hostname).Should(Equal(node1Name)) }) }) diff --git a/pkg/values/values.go b/pkg/values/values.go index 30261bd3..f488ae52 100644 --- a/pkg/values/values.go +++ b/pkg/values/values.go @@ -23,8 +23,10 @@ const ( // LabelUpgrading is the key of the upgrading label for nodes LabelUpgrading = "upgrade.openeuler.org/upgrading" // LabelMaster is the key of the master-node label for nodes - LabelMaster = "node-role.kubernetes.io/control-plane" - defaultPeriod = 15 * time.Second + LabelMaster = "node-role.kubernetes.io/control-plane" + // LabelOSinstance is used to select the osinstance with the nodeName by label + LabelOSinstance = "upgrade.openeuler.org/osinstance-node" + defaultPeriod = 15 * time.Second // OsiStatusName is param name of nodeStatus in osInstance OsiStatusName = "nodestatus" // UpgradeConfigName is param name of UpgradeConfig -- Gitee From 470e190db2de92b65e7fa720864823e7245c51e9 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Mon, 7 Aug 2023 19:09:18 +0800 Subject: [PATCH 04/27] KubeOS: fix the hostshell cannot obtain the lib Fix the hostshell cannot obtain the lib Signed-off-by: liyuanr --- cmd/admin-container/main.go | 40 +++++++++++++++++++++++++------------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/cmd/admin-container/main.go b/cmd/admin-container/main.go index f6a72939..5fa08381 100644 --- a/cmd/admin-container/main.go +++ b/cmd/admin-container/main.go @@ -22,28 +22,42 @@ import ( "github.com/sirupsen/logrus" ) +const ( + bashPath = "/usr/bin/bash" + usrBin = "/usr/bin" + usrSbin = "/usr/sbin" + localBin = "/usr/local/bin" + localSbin = "/usr/local/sbin" + usrLib = "/usr/lib" + usrLib64 = "/usr/lib64" + lib = "/lib" + lib64 = "/lib64" + envPathPrefix = "PATH=$PATH:" + envLdLibrarPathPrefix = "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:" +) + func main() { EUID := os.Geteuid() rootEUID := 0 // 0 indicates that the process has the permission of the root user. if EUID != rootEUID { logrus.Error("please use root to run hostshell") - return + } PPID := os.Getppid() rootFsPath := "/proc/" + strconv.Itoa(PPID) + "/root" - bashPath := "/usr/bin/bash" - usrBin := "/usr/bin" - usrSbin := "/usr/sbin" - localBin := "/usr/local/bin" - localSbin := "/usr/local/sbin" - paths := []string{usrBin, usrSbin, localBin, localSbin} - for i, p := range paths { - paths[i] = rootFsPath + p - } - path := "PATH=$PATH:" + strings.Join(paths, ":") - lib := "LD_LIBRARY_PATH=/lib:/lib64:/usr/lib:/usr/lib64:$LD_LIBRARY_PATH" + path := concatenateEnvPath(rootFsPath, envPathPrefix, []string{usrBin, usrSbin, localBin, localSbin}) + libPath := concatenateEnvPath(rootFsPath, envLdLibrarPathPrefix, []string{usrLib, usrLib64, lib, lib64}) if err := syscall.Exec("/usr/bin/nsenter", []string{"nsenter", "-t", "1", "-a", - "env", "-i", path, lib, rootFsPath + bashPath}, os.Environ()); err != nil { + "env", "-i", path, libPath, rootFsPath + bashPath}, os.Environ()); err != nil { logrus.Error("nsenter excute error", err) } } + +func concatenateEnvPath(prefix string, envVarPrefix string, paths []string) string { + for i, p := range paths { + paths[i] = prefix + p + } + pathLine := envVarPrefix + strings.Join(paths, ":") + pathEnv := os.ExpandEnv(pathLine) + return pathEnv +} -- Gitee From 923529f49a9f4a8d9678e087735459ea875ac5d6 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Fri, 4 Aug 2023 10:17:24 +0800 Subject: [PATCH 05/27] KubeOS: add agent, proxy and operator ut add disk_image, server, config, proxy and operator unit testing fix the bug that agent allows configuring kv with nil key Signed-off-by: Yuhang Wei --- Makefile | 4 +- cmd/agent/server/config.go | 4 + cmd/agent/server/config_test.go | 130 +++++++-- cmd/agent/server/disk_image.go | 8 +- cmd/agent/server/disk_image_test.go | 262 ++++++++++++------ cmd/agent/server/server.go | 4 + cmd/agent/server/server_test.go | 86 +++++- cmd/agent/server/utils_test.go | 146 +++++++++- .../controllers/os_controller_test.go | 67 +++++ cmd/operator/controllers/suite_test.go | 2 +- cmd/proxy/controllers/os_controller.go | 6 + cmd/proxy/controllers/os_controller_test.go | 153 +++++++++- cmd/proxy/controllers/suite_test.go | 2 +- 13 files changed, 748 insertions(+), 126 deletions(-) diff --git a/Makefile b/Makefile index fbabda6c..b5b61613 100644 --- a/Makefile +++ b/Makefile @@ -133,14 +133,14 @@ kustomize: $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) ARCH := $(shell uname -m) -TEST_CMD := go test ./... -race -count=1 -timeout=300s -cover -gcflags=all=-l +TEST_CMD := go test ./... -race -count=1 -timeout=300s -cover -gcflags=all=-l -p 1 ifeq ($(ARCH), aarch64) TEST_CMD := ETCD_UNSUPPORTED_ARCH=arm64 $(TEST_CMD) endif .PHONY: test -test: manifests fmt vet envtest ## Run tests. +test: fmt vet envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(TEST_CMD) .PHONY: envtest diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index e7110f8a..653f9132 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -374,6 +374,10 @@ func handleUpdateKey(config []string, configInfo *agent.KeyInfo, isFound bool) s func handleAddKey(m map[string]*agent.KeyInfo, isOnlyKeyValid bool) []string { var configs []string for key, keyInfo := range m { + if key == "" { + logrus.Warnln("Failed to add nil key") + continue + } if keyInfo.Operation == "delete" { logrus.Warnf("Failed to delete inexistent key %s", key) continue diff --git a/cmd/agent/server/config_test.go b/cmd/agent/server/config_test.go index 903af878..2deb15f8 100644 --- a/cmd/agent/server/config_test.go +++ b/cmd/agent/server/config_test.go @@ -67,6 +67,15 @@ func TestKernelSysctl_SetConfig(t *testing.T) { }, }}, }, + { + name: "nil value", + k: KernelSysctl{}, + args: args{config: &agent.SysConfig{ + Contents: map[string]*agent.KeyInfo{ + "d": {Value: ""}, + }, + }}, + }, } tmpDir := t.TempDir() patchGetProcPath := gomonkey.ApplyFuncReturn(getDefaultProcPath, tmpDir+"/") @@ -84,6 +93,7 @@ func TestKernelSysctl_SetConfig(t *testing.T) { func TestKerSysctlPersist_SetConfig(t *testing.T) { tmpDir := t.TempDir() persistPath := tmpDir + "/test-persist.conf" + comment := `# This file is managed by KubeOS for unit testing.` type args struct { config *agent.SysConfig } @@ -94,6 +104,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { want []string wantErr bool }{ + {name: "create file", args: args{config: &agent.SysConfig{ConfigPath: persistPath}}, want: []string{comment}, wantErr: false}, { name: "add configs", args: args{ @@ -103,12 +114,15 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { "a": {Value: "1"}, "b": {Value: "2"}, "c": {Value: ""}, + "": {Value: "4"}, + "e": {Value: "5"}, }, }, }, want: []string{ "a=1", "b=2", + "e=5", }, wantErr: false, }, @@ -126,6 +140,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { want: []string{ "a=2", "b=2", + "e=5", }, wantErr: false, }, @@ -137,11 +152,16 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { Contents: map[string]*agent.KeyInfo{ "a": {Value: "1", Operation: "delete"}, "b": {Value: "2", Operation: "delete"}, + "c": {Value: "3", Operation: "delete"}, + "e": {Value: "5", Operation: "remove"}, + "f": {Value: "6", Operation: "remove"}, }, }, }, want: []string{ "a=2", + "e=5", + "f=6", }, wantErr: false, }, @@ -152,14 +172,21 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { if err := k.SetConfig(tt.args.config); (err != nil) != tt.wantErr { t.Errorf("KerSysctlPersist.SetConfig() error = %v, wantErr %v", err, tt.wantErr) } + if tt.name == "create file" { + if err := os.WriteFile(persistPath, []byte(comment), 0644); err != nil { + t.Fatalf("failed to write file %s", persistPath) + } + } data, err := os.ReadFile(persistPath) if err != nil { t.Errorf("failed to read file %s", persistPath) } lines := strings.Split(string(data), "\n") - // remove the last empty line - lines = lines[:len(lines)-1] - sort.Strings(lines) + if tt.name != "create file" { + // remove the comment and the last empty line + lines = lines[1 : len(lines)-1] + sort.Strings(lines) + } if !reflect.DeepEqual(lines, tt.want) { t.Errorf("KerSysctlPersist file contents not equal, expect: %v, get: %v", tt.want, lines) } @@ -210,17 +237,36 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri args: args{ config: &agent.SysConfig{ Contents: map[string]*agent.KeyInfo{ - "panic": {Value: "5"}, // update existent kv - "quiet": {Value: "", Operation: "delete"}, // delete existent key - "oops": {Value: ""}, // update existent kv with null value - "selinux": {Value: "1", Operation: "delete"}, // failed to delete inconsistent kv - "acpi": {Value: "off", Operation: "delete"}, // failed to delete inexistent kv - "debug": {}, // add key - "pci": {Value: "nomis"}, // add kv + "debug": {}, // add key + "pci": {Value: "nomis"}, // add kv + "quiet": {Value: "", Operation: "delete"}, // delete existent key + "panic": {Value: "5"}, // update existent kv + "nomodeset": {Operation: "update"}, // invalid operation, default to update existent key + "softlockup_panic": {Value: "0", Operation: "update"}, // invalid operation, default to update existent kv + "oops": {Value: ""}, // warning, skip, update existent kv with null value + "": {Value: "test"}, // warning, skip, failed to add kv with empty key + "selinux": {Value: "1", Operation: "delete"}, // failed to delete inconsistent kv + "acpi": {Value: "off", Operation: "delete"}, // failed to delete inexistent kv }, }, }, - pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=1\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + wantErr: false, + }, + { + name: "delete and invalid operation", + g: GrubCmdline{isCurPartition: true}, + args: args{ + config: &agent.SysConfig{ + Contents: map[string]*agent.KeyInfo{ + "debug": {Operation: "delete"}, // delete key + "pci": {Value: "nomis", Operation: "delete"}, // delete kv + "debugpat": {Value: "", Operation: "add"}, // passed key, operation is invalid, default to add key + "audit": {Value: "1", Operation: "add"}, // passed kv, key is inexistent, operation is invalid, default to add kv + }, + }, + }, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debugpat\saudit=1|audit=1\sdebugpat)$`, wantErr: false, }, { @@ -229,23 +275,27 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri args: args{ config: &agent.SysConfig{ Contents: map[string]*agent.KeyInfo{ - "panic": {Value: "4"}, - "quiet": {Value: "", Operation: "delete"}, - "oops": {Value: ""}, // update existent kv with null value - "selinux": {Value: "1", Operation: "delete"}, - "acpi": {Value: "off", Operation: "delete"}, - "debug": {}, - "pci": {Value: "nomis"}, + "debug": {}, + "pci": {Value: "nomis"}, + "quiet": {Value: "", Operation: "delete"}, + "panic": {Value: "4"}, + "nomodeset": {Operation: "update"}, // invalid operation, default to update existent key + "softlockup_panic": {Value: "0", Operation: "update"}, // invalid operation, default to update existent kv + "oops": {Value: ""}, // update existent kv with null value + "": {Value: "test"}, // warning, skip, failed to add kv with empty key + "selinux": {Value: "1", Operation: "delete"}, + "acpi": {Value: "off", Operation: "delete"}, }, }, }, - pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=1\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=4\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=4\s+(debug\spci=nomis|pci=nomis\sdebug)$`, wantErr: false, }, } patchGetGrubPath := gomonkey.ApplyFuncReturn(getGrubCfgPath, grubCfgPath) defer patchGetGrubPath.Reset() patchGetConfigPartition := gomonkey.ApplyFuncSeq(getConfigPartition, []gomonkey.OutputCell{ + {Values: gomonkey.Params{false, nil}}, {Values: gomonkey.Params{false, nil}}, {Values: gomonkey.Params{true, nil}}, }) @@ -375,3 +425,45 @@ func Test_getGrubCfgPath(t *testing.T) { }) } } + +func Test_getConfigPartition(t *testing.T) { + type args struct { + isCurPartition bool + } + tests := []struct { + name string + args args + want bool + wantErr bool + }{ + { + name: "get current partition", + args: args{isCurPartition: true}, + want: false, + wantErr: false, + }, + { + name: "get next partition", + args: args{isCurPartition: false}, + want: true, + wantErr: false, + }, + } + patchRootfsDisks := gomonkey.ApplyFuncReturn(getRootfsDisks, "/dev/sda2", "/dev/sda3", nil) + defer patchRootfsDisks.Reset() + // assume now is partition A, want to swiching to partition B + patchGetNextPartition := gomonkey.ApplyFuncReturn(getNextPart, "/dev/sda3", "B", nil) + defer patchGetNextPartition.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getConfigPartition(tt.args.isCurPartition) + if (err != nil) != tt.wantErr { + t.Errorf("getConfigPartition() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getConfigPartition() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/agent/server/disk_image.go b/cmd/agent/server/disk_image.go index c89003b9..8bd6bf67 100644 --- a/cmd/agent/server/disk_image.go +++ b/cmd/agent/server/disk_image.go @@ -153,7 +153,7 @@ func loadCaCerts(caCert string) (*http.Client, error) { if err != nil { return &http.Client{}, err } - ca, err := ioutil.ReadFile(certPath + caCert) + ca, err := ioutil.ReadFile(getCertPath() + caCert) if err != nil { return &http.Client{}, fmt.Errorf("read the ca certificate error %s", err) } @@ -173,7 +173,7 @@ func loadClientCerts(caCert, clientCert, clientKey string) (*http.Client, error) if err != nil { return &http.Client{}, err } - ca, err := ioutil.ReadFile(certPath + caCert) + ca, err := ioutil.ReadFile(getCertPath() + caCert) if err != nil { return &http.Client{}, err } @@ -186,7 +186,7 @@ func loadClientCerts(caCert, clientCert, clientKey string) (*http.Client, error) if err != nil { return &http.Client{}, err } - cliCrt, err := tls.LoadX509KeyPair(certPath+clientCert, certPath+clientKey) + cliCrt, err := tls.LoadX509KeyPair(getCertPath()+clientCert, getCertPath()+clientKey) if err != nil { return &http.Client{}, err } @@ -206,7 +206,7 @@ func certExist(certFile string) error { if certFile == "" { return fmt.Errorf("please provide the certificate") } - _, err := os.Stat(certPath + certFile) + _, err := os.Stat(getCertPath() + certFile) if err != nil { if os.IsNotExist(err) { return fmt.Errorf("certificate is not exist %s ", err) diff --git a/cmd/agent/server/disk_image_test.go b/cmd/agent/server/disk_image_test.go index 3c821131..71c5de7f 100644 --- a/cmd/agent/server/disk_image_test.go +++ b/cmd/agent/server/disk_image_test.go @@ -14,19 +14,35 @@ package server import ( - "crypto/tls" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" "crypto/x509" + "crypto/x509/pkix" + "encoding/hex" + "encoding/pem" + "io" + "math/big" "net/http" "os" "reflect" + "strings" + "syscall" "testing" + "time" "github.com/agiledragon/gomonkey/v2" - pb "openeuler.org/KubeOS/cmd/agent/api" ) -func Testdownload(t *testing.T) { +func Test_download(t *testing.T) { + tmpDir := t.TempDir() + tmpFileForDownload := tmpDir + "/tmpFileForDownload" + tmpFile, err := os.Create(tmpFileForDownload) + if err != nil { + t.Errorf("open file error: %v", err) + } + defer tmpFile.Close() type args struct { req *pb.UpdateRequest } @@ -36,14 +52,35 @@ func Testdownload(t *testing.T) { want string wantErr bool }{ - {name: "errornil", args: args{&pb.UpdateRequest{Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, - {name: "normal", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "/persist/update.img", wantErr: false}, - {name: "errornodir", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + // {name: "errornil", args: args{&pb.UpdateRequest{Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + // {name: "normal", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "/persist/update.img", wantErr: false}, + // {name: "errornodir", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + { + name: "normal", + args: args{ + req: &pb.UpdateRequest{ + ImageUrl: "http://www.openeuler.org/zh/", + }, + }, + want: tmpFileForDownload, + wantErr: false, + }, } + patchStatfs := gomonkey.ApplyFunc(syscall.Statfs, func(path string, stat *syscall.Statfs_t) error { + stat.Bfree = 3000 + stat.Bsize = 4096 + return nil + }) + defer patchStatfs.Reset() + patchGetImageUrl := gomonkey.ApplyFuncReturn(getImageURL, &http.Response{ + StatusCode: http.StatusOK, + ContentLength: 5, + Body: io.NopCloser(strings.NewReader("hello")), + }, nil) + defer patchGetImageUrl.Reset() + patchOSCreate := gomonkey.ApplyFuncReturn(os.Create, tmpFile, nil) + defer patchOSCreate.Reset() for _, tt := range tests { - if tt.name == "normal" { - os.Mkdir("/persist", os.ModePerm) - } t.Run(tt.name, func(t *testing.T) { got, err := download(tt.args.req) if (err != nil) != tt.wantErr { @@ -54,47 +91,42 @@ func Testdownload(t *testing.T) { t.Errorf("download() got = %v, want %v", got, tt.want) } }) - if tt.name == "normal" { - os.RemoveAll("/persist") - } } } -func TestcheckSumMatch(t *testing.T) { +func Test_checkSumMatch(t *testing.T) { + tmpDir := t.TempDir() + tmpFileForCheckSum := tmpDir + "/tmpFileForCheckSum" + err := os.WriteFile(tmpFileForCheckSum, []byte("hello"), 0644) + if err != nil { + t.Errorf("open file error: %v", err) + } type args struct { filePath string checkSum string } - ff, _ := os.Create("aa.txt") - ff.Chmod(os.ModePerm) tests := []struct { name string args args wantErr bool }{ - {name: "error", args: args{filePath: "aaa", checkSum: "aaa"}, wantErr: true}, - {name: "errordir", args: args{filePath: "/aaa", checkSum: "/aaa"}, wantErr: true}, - {name: "errortxt", args: args{filePath: "aa.txt", checkSum: "aa.txt"}, wantErr: true}, + { + name: "normal", + args: args{filePath: tmpFileForCheckSum, checkSum: calculateChecksum("hello")}, + wantErr: false, + }, + {name: "error", args: args{filePath: tmpFileForCheckSum, checkSum: "aaa"}, wantErr: true}, } for _, tt := range tests { - if tt.name == "errordir" { - os.Mkdir("/aaa", os.ModePerm) - } t.Run(tt.name, func(t *testing.T) { if err := checkSumMatch(tt.args.filePath, tt.args.checkSum); (err != nil) != tt.wantErr { t.Errorf("checkSumMatch() error = %v, wantErr %v", err, tt.wantErr) } }) - if tt.name == "errordir" { - os.RemoveAll("/aaa") - } } - defer os.Remove("aa.txt") - defer ff.Close() - } -func TestgetImageURL(t *testing.T) { +func Test_getImageURL(t *testing.T) { type args struct { req *pb.UpdateRequest } @@ -105,23 +137,29 @@ func TestgetImageURL(t *testing.T) { wantErr bool }{ {name: "httpNotSafe", args: args{req: &pb.UpdateRequest{ - ImageUrl: "http://www.openeuler.org/zh/", + ImageUrl: "http://www.openeuler.abc/zh/", FlagSafe: false, MTLS: false, Certs: &pb.CertsInfo{}, }}, want: &http.Response{}, wantErr: true}, - {name: "mTLSError", args: args{req: &pb.UpdateRequest{ - ImageUrl: "http://www.openeuler.org/zh/", + {name: "httpSuccess", args: args{req: &pb.UpdateRequest{ + ImageUrl: "http://www.openeuler.abc/zh/", + FlagSafe: true, + MTLS: false, + Certs: &pb.CertsInfo{}, + }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, + {name: "mTLSGetSuccess", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", FlagSafe: true, MTLS: true, Certs: &pb.CertsInfo{}, - }}, want: &http.Response{}, wantErr: true}, - {name: "httpsError", args: args{req: &pb.UpdateRequest{ - ImageUrl: "https://www.openeuler.org/zh/", + }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, + {name: "httpsGetSuccess", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", FlagSafe: true, MTLS: false, Certs: &pb.CertsInfo{}, - }}, want: &http.Response{}, wantErr: true}, + }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, } patchLoadClientCerts := gomonkey.ApplyFunc(loadClientCerts, func(caCert, clientCert, clientKey string) (*http.Client, error) { return &http.Client{}, nil @@ -131,8 +169,20 @@ func TestgetImageURL(t *testing.T) { return &http.Client{}, nil }) defer patchLoadCaCerts.Reset() + patchGet := gomonkey.ApplyFunc(http.Get, func(url string) (resp *http.Response, err error) { + return &http.Response{StatusCode: http.StatusOK}, nil + }) + defer patchGet.Reset() + patchClientGet := gomonkey.ApplyMethod(reflect.TypeOf(&http.Client{}), "Get", func(_ *http.Client, url string) (resp *http.Response, err error) { + return &http.Response{StatusCode: http.StatusOK}, nil + }) + defer patchClientGet.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.name == "httpSuccess" { + patchGet := gomonkey.ApplyFuncReturn(http.Get, &http.Response{StatusCode: http.StatusOK}, nil) + defer patchGet.Reset() + } got, err := getImageURL(tt.args.req) if (err != nil) != tt.wantErr { t.Errorf("getImageURL() error = %v, wantErr %v", err, tt.wantErr) @@ -145,20 +195,28 @@ func TestgetImageURL(t *testing.T) { } } -func TestloadCaCerts(t *testing.T) { +func Test_loadCaCerts(t *testing.T) { + tmpDir := t.TempDir() + caPath := tmpDir + "/fake.crt" + createFakeCertKey(caPath, "") type args struct { caCert string } tests := []struct { name string args args - want *http.Client wantErr bool }{ - {name: "noCaCertError", args: args{caCert: "bb.txt"}, want: &http.Client{}, wantErr: true}, + { + name: "normal", + args: args{ + caCert: caPath, + }, + wantErr: false, + }, } - os.MkdirAll(certPath, 0644) - defer os.RemoveAll(certPath) + patchGetCertPath := gomonkey.ApplyFuncReturn(getCertPath, "") + defer patchGetCertPath.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := loadCaCerts(tt.args.caCert) @@ -166,48 +224,39 @@ func TestloadCaCerts(t *testing.T) { t.Errorf("loadCaCerts() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("loadCaCerts() = %v, want %v", got, tt.want) + if got == nil { + t.Errorf("loadCaCerts() = %v", got) } }) } } -func TestloadClientCerts(t *testing.T) { +func Test_loadClientCerts(t *testing.T) { + tmpDir := t.TempDir() + clientCertPath := tmpDir + "/fakeClientCert.crt" + clientKeyPath := tmpDir + "/fakeClientKey.crt" + createFakeCertKey(clientCertPath, clientKeyPath) type args struct { caCert string clientCert string clientKey string } - pool := &x509.CertPool{} tests := []struct { name string args args - want *http.Client wantErr bool }{ - {name: "noCaCertError", args: args{" dd.txt", "bb.txt", "cc.txt"}, want: &http.Client{}, wantErr: true}, - {name: "noClientCertError", args: args{"ca.crt", "bb.txt", "cc.txt"}, want: &http.Client{}, wantErr: true}, - {name: "noClientKeyError", args: args{"ca.crt", "client.crt", "cc.txt"}, want: &http.Client{}, wantErr: true}, - } - os.MkdirAll(certPath, 0644) - caFile, _ := os.Create(certPath + "ca.crt") - clientCertFile, _ := os.Create(certPath + "client.crt") - clientKeyFile, _ := os.Create(certPath + "client.key") - - patchNewCertPool := gomonkey.ApplyFunc(x509.NewCertPool, func() *x509.CertPool { - return pool - }) - defer patchNewCertPool.Reset() - patchAppendCertsFromPEM := gomonkey.ApplyMethod(reflect.TypeOf(pool), "AppendCertsFromPEM", func(_ *x509.CertPool, _ []byte) (ok bool) { - return true - }) - defer patchAppendCertsFromPEM.Reset() - patchLoadX509KeyPair := gomonkey.ApplyFunc(tls.LoadX509KeyPair, func(certFile string, keyFile string) (tls.Certificate, error) { - return tls.Certificate{}, nil - }) - defer patchLoadX509KeyPair.Reset() + { + name: "normal", + args: args{ + caCert: clientCertPath, clientCert: clientCertPath, clientKey: clientKeyPath, + }, + wantErr: false, + }, + } + patchGetCertPath := gomonkey.ApplyFuncReturn(getCertPath, "") + defer patchGetCertPath.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := loadClientCerts(tt.args.caCert, tt.args.clientCert, tt.args.clientKey) @@ -215,19 +264,14 @@ func TestloadClientCerts(t *testing.T) { t.Errorf("loadClientCerts() error = %v, wantErr %v", err, tt.wantErr) return } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("loadClientCerts() got = %v, want %v", got, tt.want) + if got == nil { + t.Errorf("loadClientCerts() got = %v", got) } }) } - caFile.Close() - clientCertFile.Close() - clientKeyFile.Close() - defer os.RemoveAll("/etc/KubeOS") - } -func TestcertExist(t *testing.T) { +func Test_certExist(t *testing.T) { type args struct { certFile string } @@ -238,10 +282,7 @@ func TestcertExist(t *testing.T) { }{ {name: "fileEmpty", args: args{certFile: ""}, wantErr: true}, {name: "fileNotExist", args: args{certFile: "bb.txt"}, wantErr: true}, - {name: "normal", args: args{certFile: "aa.txt"}, wantErr: false}, } - os.MkdirAll(certPath, 0644) - ff, _ := os.Create(certPath + "aa.txt") for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := certExist(tt.args.certFile); (err != nil) != tt.wantErr { @@ -249,6 +290,71 @@ func TestcertExist(t *testing.T) { } }) } - ff.Close() defer os.RemoveAll("/etc/KubeOS/") } + +func createFakeCertKey(certPath, keyPath string) { + privateKey, _ := rsa.GenerateKey(rand.Reader, 2048) + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "Fake Client Certificate", + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(1, 0, 0), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}, + BasicConstraintsValid: true, + } + certBytes, _ := x509.CreateCertificate(rand.Reader, &template, &template, &privateKey.PublicKey, privateKey) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certBytes}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(privateKey)}) + os.WriteFile(certPath, certPEM, 0644) + if keyPath != "" { + os.WriteFile(keyPath, keyPEM, 0644) + } +} + +func calculateChecksum(data string) string { + hash := sha256.New() + hash.Write([]byte(data)) + return hex.EncodeToString(hash.Sum(nil)) +} + +func Test_diskHandler_getRootfsArchive(t *testing.T) { + type args struct { + req *pb.UpdateRequest + neededPath preparePath + } + tests := []struct { + name string + d diskHandler + args args + want string + wantErr bool + }{ + { + name: "normal", d: diskHandler{}, + args: args{req: &pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/"}, neededPath: preparePath{}}, + want: "/persist/update.img", + wantErr: false, + }, + } + patchDownload := gomonkey.ApplyFuncReturn(download, "/persist/update.img", nil) + defer patchDownload.Reset() + patchCheckSumMatch := gomonkey.ApplyFuncReturn(checkSumMatch, nil) + defer patchCheckSumMatch.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := diskHandler{} + got, err := d.getRootfsArchive(tt.args.req, tt.args.neededPath) + if (err != nil) != tt.wantErr { + t.Errorf("diskHandler.getRootfsArchive() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("diskHandler.getRootfsArchive() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/agent/server/server.go b/cmd/agent/server/server.go index b41ebc49..8ac6ffd6 100644 --- a/cmd/agent/server/server.go +++ b/cmd/agent/server/server.go @@ -171,3 +171,7 @@ func (s *Server) reboot() error { } return syscall.Reboot(syscall.LINUX_REBOOT_CMD_RESTART) } + +func getCertPath() string { + return certPath +} diff --git a/cmd/agent/server/server_test.go b/cmd/agent/server/server_test.go index 0aac36ab..74e2eade 100644 --- a/cmd/agent/server/server_test.go +++ b/cmd/agent/server/server_test.go @@ -89,7 +89,21 @@ func TestServerUpdate(t *testing.T) { {name: "error", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, args: args{in0: context.Background(), req: &pb.UpdateRequest{Version: "test", Certs: &pb.CertsInfo{}}}, want: &pb.UpdateResponse{}, wantErr: true}, + {name: "success", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, + args: args{in0: context.Background(), req: &pb.UpdateRequest{Version: "test", Certs: &pb.CertsInfo{}, ImageType: "containerd"}}, + want: &pb.UpdateResponse{}, wantErr: false}, } + patchRootfsDisks := gomonkey.ApplyFuncReturn(getRootfsDisks, "/dev/sda2", "/dev/sda3", nil) + defer patchRootfsDisks.Reset() + // assume now is partition A, want to swiching to partition B + patchGetNextPartition := gomonkey.ApplyFuncReturn(getNextPart, "/dev/sda3", "B", nil) + defer patchGetNextPartition.Reset() + patchDownloadImage := gomonkey.ApplyPrivateMethod(conImageHandler{}, "downloadImage", func(_ conImageHandler, req *pb.UpdateRequest) (string, error) { + return "", nil + }) + defer patchDownloadImage.Reset() + patchInstall := gomonkey.ApplyFuncReturn(install, nil) + defer patchInstall.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &Server{ @@ -129,11 +143,26 @@ func TestServerRollback(t *testing.T) { {name: "error", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, args: args{in0: context.Background(), req: &pb.RollbackRequest{}}, want: &pb.RollbackResponse{}, wantErr: true}, + {name: "success", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, + args: args{in0: context.Background(), req: &pb.RollbackRequest{}}, + want: &pb.RollbackResponse{}, wantErr: false}, } - patchGetNextPart := gomonkey.ApplyFunc(getNextPart, func(partA string, partB string) (string, string, error) { - return "", "", fmt.Errorf("rollbak test error") + patchRootfsDisks := gomonkey.ApplyFuncReturn(getRootfsDisks, "/dev/sda2", "/dev/sda3", nil) + defer patchRootfsDisks.Reset() + // assume now is partition A, want to swiching to partition B + patchGetNextPartition := gomonkey.ApplyFuncSeq(getNextPart, []gomonkey.OutputCell{ + {Values: gomonkey.Params{"", "", fmt.Errorf("rollbak test error")}}, + {Values: gomonkey.Params{"/dev/sda3", "B", nil}}, }) - defer patchGetNextPart.Reset() + defer patchGetNextPartition.Reset() + patchDownloadImage := gomonkey.ApplyPrivateMethod(conImageHandler{}, "downloadImage", func(_ conImageHandler, req *pb.UpdateRequest) (string, error) { + return "", nil + }) + defer patchDownloadImage.Reset() + patchInstall := gomonkey.ApplyFuncReturn(install, nil) + defer patchInstall.Reset() + patchRunCommand := gomonkey.ApplyFuncReturn(runCommand, nil) + defer patchRunCommand.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { s := &Server{ @@ -179,9 +208,9 @@ func TestServerupdate(t *testing.T) { }}, wantErr: true}, {name: "errordocker", args: args{&pb.UpdateRequest{ - DockerImage: "", - ImageType: "docker", - Certs: &pb.CertsInfo{}, + ContainerImage: "", + ImageType: "docker", + Certs: &pb.CertsInfo{}, }}, wantErr: true}, } @@ -264,3 +293,48 @@ func TestServerreboot(t *testing.T) { }) } } + +func TestServer_Configure(t *testing.T) { + type fields struct { + UnimplementedOSServer pb.UnimplementedOSServer + mutex Lock + disableReboot bool + } + type args struct { + in0 context.Context + req *pb.ConfigureRequest + } + tests := []struct { + name string + fields fields + args args + want *pb.ConfigureResponse + wantErr bool + }{ + // TODO: Add test cases. + { + name: "nil", + fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, + args: args{in0: context.Background(), req: &pb.ConfigureRequest{}}, + want: &pb.ConfigureResponse{}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &Server{ + UnimplementedOSServer: tt.fields.UnimplementedOSServer, + mutex: tt.fields.mutex, + disableReboot: tt.fields.disableReboot, + } + got, err := s.Configure(tt.args.in0, tt.args.req) + if (err != nil) != tt.wantErr { + t.Errorf("Server.Configure() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("Server.Configure() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/agent/server/utils_test.go b/cmd/agent/server/utils_test.go index 8e7fd908..89b2c3bf 100644 --- a/cmd/agent/server/utils_test.go +++ b/cmd/agent/server/utils_test.go @@ -14,13 +14,17 @@ package server import ( + "archive/tar" "os" "os/exec" - "strings" + "reflect" "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" ) -func TestrunCommand(t *testing.T) { +func Test_runCommand(t *testing.T) { type args struct { name string args []string @@ -41,23 +45,21 @@ func TestrunCommand(t *testing.T) { } } -func Testinstall(t *testing.T) { +func Test_install(t *testing.T) { type args struct { imagePath string side string next string } - out, _ := exec.Command("bash", "-c", "df -h | grep '/$' | awk '{print $1}'").CombinedOutput() - mountPart := strings.TrimSpace(string(out)) tests := []struct { name string args args wantErr bool }{ - {name: "normal", args: args{imagePath: "aa.txt", side: mountPart, next: ""}, wantErr: false}, + {name: "normal", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: false}, } - ff, _ := os.Create("aa.txt") - ff.Chmod(os.ModePerm) + patchRunCommand := gomonkey.ApplyFuncReturn(runCommand, nil) + defer patchRunCommand.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := install(tt.args.imagePath, tt.args.side, tt.args.next); (err != nil) != tt.wantErr { @@ -65,17 +67,13 @@ func Testinstall(t *testing.T) { } }) } - ff.Close() - defer os.Remove("aa.txt") } -func TestgetNextPart(t *testing.T) { +func Test_getNextPart(t *testing.T) { type args struct { partA string partB string } - out, _ := exec.Command("bash", "-c", "df -h | grep '/$' | awk '{print $1}'").CombinedOutput() - mountPart := strings.TrimSpace(string(out)) tests := []struct { name string args args @@ -83,8 +81,14 @@ func TestgetNextPart(t *testing.T) { want1 string wantErr bool }{ - {name: "normal", args: args{partA: mountPart, partB: "testB"}, want: "testB", want1: "B", wantErr: false}, + {name: "switch to sda3", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "/dev/sda3", want1: "B", wantErr: false}, + {name: "switch to sda2", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "/dev/sda2", want1: "A", wantErr: false}, } + patchExecCommand := gomonkey.ApplyMethodSeq(&exec.Cmd{}, "CombinedOutput", []gomonkey.OutputCell{ + {Values: gomonkey.Params{[]byte("/"), nil}}, + {Values: gomonkey.Params{[]byte(""), nil}}, + }) + defer patchExecCommand.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, got1, err := getNextPart(tt.args.partA, tt.args.partB) @@ -101,3 +105,117 @@ func TestgetNextPart(t *testing.T) { }) } } + +func Test_prepareEnv(t *testing.T) { + mountPath := "/persist/KubeOS-Update/kubeos-update" + if err := os.MkdirAll(mountPath, 0644); err != nil { + t.Fatalf("mkdir err %v", err) + } + defer os.RemoveAll("/persist") + tests := []struct { + name string + want preparePath + wantErr bool + }{ + { + name: "success", + want: preparePath{ + updatePath: "/persist/KubeOS-Update", + mountPath: "/persist/KubeOS-Update/kubeos-update", + tarPath: "/persist/KubeOS-Update/os.tar", + imagePath: "/persist/update.img", + rootfsFile: "os.tar", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := prepareEnv() + if (err != nil) != tt.wantErr { + t.Errorf("prepareEnv() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("prepareEnv() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_createOSImage(t *testing.T) { + mountPath := "/persist/KubeOS-Update/kubeos-update" + if err := os.MkdirAll(mountPath, 0644); err != nil { + t.Fatalf("mkdir err %v", err) + } + defer os.RemoveAll("/persist") + tarPath := "/persist/KubeOS-Update/os.tar" + path, err := createTmpTarFile(tarPath) + if path != tarPath && err != nil { + t.Fatalf("create temp zip file err %v", err) + } + type args struct { + neededPath preparePath + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "normal", + args: args{ + neededPath: preparePath{ + updatePath: "/persist/KubeOS-Update", + mountPath: "/persist/KubeOS-Update/kubeos-update", + tarPath: "/persist/KubeOS-Update/os.tar", + imagePath: "/persist/update.img", + }, + }, + want: "/persist/update.img", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := createOSImage(tt.args.neededPath) + if (err != nil) != tt.wantErr { + t.Errorf("createOSImage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("createOSImage() = %v, want %v", got, tt.want) + } + }) + } +} + +func createTmpTarFile(tarPath string) (string, error) { + tempFile, err := os.Create(tarPath) + if err != nil { + return "", err + } + defer tempFile.Close() + + tarWriter := tar.NewWriter(tempFile) + fakeData := []byte("This is a fake file") + fakeFile := "fakefile.txt" + header := &tar.Header{ + Name: fakeFile, + Size: int64(len(fakeData)), + Mode: 0644, + ModTime: time.Now(), + } + + if err = tarWriter.WriteHeader(header); err != nil { + return "", err + } + if _, err := tarWriter.Write(fakeData); err != nil { + return "", err + } + if err := tarWriter.Flush(); err != nil { + return "", err + } + return tempFile.Name(), nil +} diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index a3910057..98de6d02 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -14,16 +14,22 @@ package controllers import ( "context" + "testing" "time" + "github.com/agiledragon/gomonkey/v2" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" upgradev1 "openeuler.org/KubeOS/api/v1alpha1" "openeuler.org/KubeOS/pkg/values" @@ -506,3 +512,64 @@ var _ = Describe("OsController", func() { }) }) }) + +func TestOSReconciler_DeleteOSInstance(t *testing.T) { + type fields struct { + Scheme *runtime.Scheme + Client client.Client + } + kClient, _ := client.New(cfg, client.Options{Scheme: scheme.Scheme}) + type args struct { + e event.DeleteEvent + q workqueue.RateLimitingInterface + } + tests := []struct { + name string + fields fields + args args + }{ + { + name: "delete osinstance", + fields: fields{ + Scheme: nil, + Client: kClient, + }, + args: args{ + e: event.DeleteEvent{ + Object: &upgradev1.OSInstance{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node1", + Namespace: "test", + }, + }, + }, + q: nil, + }, + }, + } + var patchList *gomonkey.Patches + var patchDelete *gomonkey.Patches + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &OSReconciler{ + Scheme: tt.fields.Scheme, + Client: tt.fields.Client, + } + patchList = gomonkey.ApplyMethodFunc(r.Client, "List", func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + list.(*upgradev1.OSInstanceList).Items = []upgradev1.OSInstance{ + { + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node1", + Namespace: "test", + }, + }, + } + return nil + }) + patchDelete = gomonkey.ApplyMethodReturn(r.Client, "Delete", nil) + r.DeleteOSInstance(tt.args.e, tt.args.q) + }) + } + defer patchDelete.Reset() + defer patchList.Reset() +} diff --git a/cmd/operator/controllers/suite_test.go b/cmd/operator/controllers/suite_test.go index 889789eb..aa6deeae 100644 --- a/cmd/operator/controllers/suite_test.go +++ b/cmd/operator/controllers/suite_test.go @@ -51,7 +51,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "docs", "example", "config", "crd")}, ErrorIfCRDPathMissing: true, } diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index b0b17e73..09d61c0d 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -87,6 +87,9 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re sameOSVersion := checkVersion(osCr.Spec.OSVersion, node.Status.NodeInfo.OSImage) if sameOSVersion { configOps, err := checkConfigVersion(osCr, osInstance, values.SysConfigName) + if err != nil { + return values.RequeueNow, err + } if configOps == values.Reassign { if err = r.refreshNode(ctx, &node, osInstance, osCr.Spec.SysConfigs.Version, values.SysConfigName); err != nil { return values.RequeueNow, err @@ -113,6 +116,9 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re return values.RequeueNow, err } configOps, err := checkConfigVersion(osCr, osInstance, values.UpgradeConfigName) + if err != nil { + return values.RequeueNow, err + } if configOps == values.Reassign { if err = r.refreshNode(ctx, &node, osInstance, osCr.Spec.UpgradeConfigs.Version, values.UpgradeConfigName); err != nil { diff --git a/cmd/proxy/controllers/os_controller_test.go b/cmd/proxy/controllers/os_controller_test.go index e6cd5b7d..d63f176e 100644 --- a/cmd/proxy/controllers/os_controller_test.go +++ b/cmd/proxy/controllers/os_controller_test.go @@ -67,6 +67,148 @@ var _ = Describe("OsController", func() { testNamespace = existingNamespace.Name }) + Context("When we want to rollback", func() { + It("Should be able to rollback to previous version", func() { + ctx := context.Background() + + By("Creating a worker node") + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + values.LabelUpgrading: "", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v2", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + reconciler.hostName = node1Name + + By("Creating the corresponding OSInstance") + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + NodeStatus: values.NodeStatusUpgrade.String(), + SysConfigs: upgradev1.SysConfigs{}, + UpgradeConfigs: upgradev1.SysConfigs{}, + }, + Status: upgradev1.OSInstanceStatus{}, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + + // stub r.Connection.RollbackSpec() + patchRollback := gomonkey.ApplyMethodReturn(reconciler.Connection, "RollbackSpec", nil) + defer patchRollback.Reset() + patchConfigure := gomonkey.ApplyMethodReturn(reconciler.Connection, "ConfigureSpec", nil) + defer patchConfigure.Reset() + + By("Creating a OS custom resource") + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "rollback", + MaxUnavailable: 3, + OSVersion: "KubeOS v1", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) + Expect(createdOS.Spec.OpsType).Should(Equal("rollback")) + + By("Changing the nodeinfo OSImage to previous version, pretending the rollback success") + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + existingNode.Status.NodeInfo.OSImage = "KubeOS v1" + Expect(k8sClient.Status().Update(ctx, existingNode)).Should(Succeed()) + + By("Changing the OS Spec config to trigger reconcile") + createdOS = &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOS)).Should(Succeed()) + + time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + // NodeStatus changes to idle then operator can reassign configs to this node + Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(false)) + }) + }) + Context("When we have a sysconfig whose version is different from current OSInstance config version", func() { It("Should configure the node", func() { ctx := context.Background() @@ -482,6 +624,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusConfig.String(), @@ -634,6 +779,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusUpgrade.String(), @@ -798,6 +946,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ NodeStatus: values.NodeStatusUpgrade.String(), @@ -915,7 +1066,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) By("Checking the OSInstance status config version failed to be updated") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { diff --git a/cmd/proxy/controllers/suite_test.go b/cmd/proxy/controllers/suite_test.go index a52c18f5..00eebbf4 100644 --- a/cmd/proxy/controllers/suite_test.go +++ b/cmd/proxy/controllers/suite_test.go @@ -53,7 +53,7 @@ var _ = BeforeSuite(func() { By("bootstrapping test environment") testEnv = &envtest.Environment{ - CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd")}, + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "docs", "example", "config", "crd")}, ErrorIfCRDPathMissing: true, } -- Gitee From 5c5922a922f225a9ebc5f99a5008ddcf182b7da6 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 10 Aug 2023 17:24:50 +0800 Subject: [PATCH 06/27] KubeOS: fix validate image name bug fix the bug that incorrectly granted illegal image names path:tag@sha256 Signed-off-by: Yuhang Wei --- cmd/agent/server/utils.go | 2 +- cmd/agent/server/utils_test.go | 109 ++++++++++++++++++++++++++++++++- 2 files changed, 109 insertions(+), 2 deletions(-) diff --git a/cmd/agent/server/utils.go b/cmd/agent/server/utils.go index 7134d74f..b42db18f 100644 --- a/cmd/agent/server/utils.go +++ b/cmd/agent/server/utils.go @@ -316,7 +316,7 @@ func isCommandAvailable(name string) bool { } func isValidImageName(image string) error { - pattern := `^((?:[\w.-]+)(?::\d+)?\/)*(?:[\w.-]+)(?::[\w_.-]+)?(?:@sha256:[a-fA-F0-9]+)?$` + pattern := `^((?:[\w.-]+)(?::\d+)?\/)*(?:[\w.-]+)((?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$` regEx, err := regexp.Compile(pattern) if err != nil { return err diff --git a/cmd/agent/server/utils_test.go b/cmd/agent/server/utils_test.go index 89b2c3bf..0796bce4 100644 --- a/cmd/agent/server/utils_test.go +++ b/cmd/agent/server/utils_test.go @@ -56,10 +56,16 @@ func Test_install(t *testing.T) { args args wantErr bool }{ - {name: "normal", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: false}, + {name: "normal uefi", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: false}, + {name: "normal legacy", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: false}, } patchRunCommand := gomonkey.ApplyFuncReturn(runCommand, nil) defer patchRunCommand.Reset() + patchGetBootMode := gomonkey.ApplyFuncSeq(getBootMode, []gomonkey.OutputCell{ + {Values: gomonkey.Params{"uefi", nil}}, + {Values: gomonkey.Params{"legacy", nil}}, + }) + defer patchGetBootMode.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { if err := install(tt.args.imagePath, tt.args.side, tt.args.next); (err != nil) != tt.wantErr { @@ -219,3 +225,104 @@ func createTmpTarFile(tarPath string) (string, error) { } return tempFile.Name(), nil } + +func Test_getBootMode(t *testing.T) { + tests := []struct { + name string + want string + wantErr bool + }{ + { + name: "uefi", + want: "uefi", + wantErr: false, + }, + { + name: "legacy", + want: "legacy", + wantErr: false, + }, + } + patchOSStat := gomonkey.ApplyFuncSeq(os.Stat, []gomonkey.OutputCell{ + {Values: gomonkey.Params{nil, nil}}, + {Values: gomonkey.Params{nil, os.ErrNotExist}}, + }) + defer patchOSStat.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getBootMode() + if (err != nil) != tt.wantErr { + t.Errorf("getBootMode() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getBootMode() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_isValidImageName(t *testing.T) { + type args struct { + image string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "valid", args: args{image: "alpine"}, wantErr: false}, + {name: "valid", args: args{image: "alpine:latest"}, wantErr: false}, + {name: "valid", args: args{image: "localhost:1234/test"}, wantErr: false}, + {name: "valid", args: args{image: "alpine:3.7"}, wantErr: false}, + {name: "valid", args: args{image: "docker.example.edu/gmr/alpine:3.7"}, wantErr: false}, + {name: "valid", args: args{image: "docker.example.com:5000/gmr/alpine@sha256:11111111111111111111111111111111"}, wantErr: false}, + {name: "valid", args: args{image: "registry.dobby.org/dobby/dobby-servers/arthound:2019-08-08"}, wantErr: false}, + {name: "valid", args: args{image: "registry.dobby.org/dobby/dobby-servers/lerphound:latest"}, wantErr: false}, + {name: "valid", args: args{image: "registry.dobby.org/dobby/dobby-servers/loophole@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8"}, wantErr: false}, + {name: "valid", args: args{image: "sosedoff/pgweb@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8"}, wantErr: false}, + {name: "valid", args: args{image: "registry.dobby.org/dobby/antique-penguin:release-production"}, wantErr: false}, + {name: "valid", args: args{image: "dalprodictus/halcon:6.7.5"}, wantErr: false}, + {name: "valid", args: args{image: "antigua/antigua:v31"}, wantErr: false}, + {name: "invalid ;", args: args{image: "alpine;v1.0"}, wantErr: true}, + {name: "invalid tag and digest1", args: args{image: "alpine:latest@sha256:11111111111111111111111111111111"}, wantErr: true}, + {name: "invalid |", args: args{image: "alpine|v1.0"}, wantErr: true}, + {name: "invalid &", args: args{image: "alpine&v1.0"}, wantErr: true}, + {name: "invalid tag and digest2", args: args{image: "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8"}, wantErr: true}, + {name: "invalid tag and digest3", args: args{image: "192.168.122.123:5000/kubeos_uefi-x86_64:euleros_v2_docker-2023-01@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := isValidImageName(tt.args.image); (err != nil) != tt.wantErr { + t.Errorf("isValidImageName() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_checkOCIImageDigestMatch(t *testing.T) { + type args struct { + containerRuntime string + imageName string + checkSum string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "invalid container runtion", args: args{containerRuntime: "dockctl", imageName: "docker.io/library/hello-world:latest", checkSum: "1abf18abf9bf9baa0a4a38d1afad4abf0d7da4544e163186e036c906c09c94fe"}, wantErr: true}, + {name: "nil image digets", args: args{containerRuntime: "crictl", imageName: "docker.io/library/hello-world:latest", checkSum: "1abf18abf9bf9baa0a4a38d1afad4abf0d7da4544e163186e036c906c09c94fe"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.name == "nil image digets" { + patchGetOCIImageDigest := gomonkey.ApplyFuncReturn(getOCIImageDigest, "", nil) + defer patchGetOCIImageDigest.Reset() + } + if err := checkOCIImageDigestMatch(tt.args.containerRuntime, tt.args.imageName, tt.args.checkSum); (err != nil) != tt.wantErr { + t.Errorf("checkOCIImageDigestMatch() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} -- Gitee From 911fbd04ba55a2560e8da8595ff379a742e5fa30 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 10 Aug 2023 17:47:54 +0800 Subject: [PATCH 07/27] KubeOS: fix a bug that failed to parse key with = Signed-off-by: Yuhang Wei --- cmd/agent/server/config.go | 6 ++-- cmd/agent/server/config_test.go | 55 +++++++++++++++++++++++++++++---- 2 files changed, 52 insertions(+), 9 deletions(-) diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index 653f9132..bcd9fde1 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -259,7 +259,7 @@ func getAndSetConfigsFromFile(expectConfigs map[string]*agent.KeyInfo, path stri configsWrite = append(configsWrite, line) continue } - configKV := strings.Split(line, "=") + configKV := strings.SplitN(line, "=", kvPair) if len(configKV) != kvPair { logrus.Errorf("could not parse systctl config %s", line) return nil, fmt.Errorf("could not parse systctl config %s", line) @@ -374,8 +374,8 @@ func handleUpdateKey(config []string, configInfo *agent.KeyInfo, isFound bool) s func handleAddKey(m map[string]*agent.KeyInfo, isOnlyKeyValid bool) []string { var configs []string for key, keyInfo := range m { - if key == "" { - logrus.Warnln("Failed to add nil key") + if key == "" || strings.Contains(key, "=") { + logrus.Warnf("Failed to add nil key or key containing =, key: %s", key) continue } if keyInfo.Operation == "delete" { diff --git a/cmd/agent/server/config_test.go b/cmd/agent/server/config_test.go index 2deb15f8..ae2a2cf8 100644 --- a/cmd/agent/server/config_test.go +++ b/cmd/agent/server/config_test.go @@ -23,7 +23,6 @@ import ( "testing" "github.com/agiledragon/gomonkey/v2" - agent "openeuler.org/KubeOS/cmd/agent/api" ) @@ -105,17 +104,27 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { wantErr bool }{ {name: "create file", args: args{config: &agent.SysConfig{ConfigPath: persistPath}}, want: []string{comment}, wantErr: false}, + { + name: "nil path", + args: args{ + config: &agent.SysConfig{}, + }, + want: []string{}, + wantErr: false, + }, { name: "add configs", args: args{ config: &agent.SysConfig{ ConfigPath: persistPath, Contents: map[string]*agent.KeyInfo{ - "a": {Value: "1"}, - "b": {Value: "2"}, - "c": {Value: ""}, - "": {Value: "4"}, - "e": {Value: "5"}, + "a": {Value: "1"}, + "b": {Value: "2"}, + "c": {Value: ""}, + "": {Value: "4"}, + "e": {Value: "5"}, + "y=1": {Value: "26"}, + "z": {Value: "x=1"}, }, }, }, @@ -123,6 +132,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { "a=1", "b=2", "e=5", + "z=x=1", }, wantErr: false, }, @@ -134,6 +144,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { Contents: map[string]*agent.KeyInfo{ "a": {Value: "2"}, "b": {Value: ""}, + "z": {Value: "x=2"}, }, }, }, @@ -141,6 +152,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { "a=2", "b=2", "e=5", + "z=x=2", }, wantErr: false, }, @@ -155,6 +167,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { "c": {Value: "3", Operation: "delete"}, "e": {Value: "5", Operation: "remove"}, "f": {Value: "6", Operation: "remove"}, + "z": {Value: "x=2", Operation: "delete"}, }, }, }, @@ -166,6 +179,8 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { wantErr: false, }, } + patchGetKernelConPath := gomonkey.ApplyFuncReturn(getKernelConPath, persistPath) + defer patchGetKernelConPath.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { k := KerSysctlPersist{} @@ -467,3 +482,31 @@ func Test_getConfigPartition(t *testing.T) { }) } } + +func Test_ConfigFactoryTemplate(t *testing.T) { + type args struct { + configType string + config *agent.SysConfig + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "error", + args: args{ + configType: "test", + config: &agent.SysConfig{}, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := ConfigFactoryTemplate(tt.args.configType, tt.args.config); (err != nil) != tt.wantErr { + t.Errorf("ConfigFactoryTemplate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} -- Gitee From 59473394cca1e227ec579236d71ec54deb79b068 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 10 Aug 2023 20:49:39 +0800 Subject: [PATCH 08/27] KubeOS: add warning log during config add warning log when configuring kv with unknown operation Signed-off-by: Yuhang Wei --- cmd/agent/server/config.go | 4 ++++ cmd/agent/server/config_test.go | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index bcd9fde1..78dfd017 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -382,6 +382,10 @@ func handleAddKey(m map[string]*agent.KeyInfo, isOnlyKeyValid bool) []string { logrus.Warnf("Failed to delete inexistent key %s", key) continue } + if keyInfo.Operation != "" { + logrus.Warnf("Unknown operation %s, adding key %s with value %s by default", + keyInfo.Operation, key, keyInfo.Value) + } k, v := strings.TrimSpace(key), strings.TrimSpace(keyInfo.Value) if keyInfo.Value == "" && isOnlyKeyValid { logrus.Infoln("add configuration ", k) diff --git a/cmd/agent/server/config_test.go b/cmd/agent/server/config_test.go index ae2a2cf8..6424885b 100644 --- a/cmd/agent/server/config_test.go +++ b/cmd/agent/server/config_test.go @@ -122,7 +122,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { "b": {Value: "2"}, "c": {Value: ""}, "": {Value: "4"}, - "e": {Value: "5"}, + "e": {Value: "5", Operation: "xxx"}, "y=1": {Value: "26"}, "z": {Value: "x=1"}, }, @@ -144,7 +144,7 @@ func TestKerSysctlPersist_SetConfig(t *testing.T) { Contents: map[string]*agent.KeyInfo{ "a": {Value: "2"}, "b": {Value: ""}, - "z": {Value: "x=2"}, + "z": {Value: "x=2", Operation: "zzz"}, }, }, }, -- Gitee From 38ed69d2ad089d124ee72a4431fcc96ef3d9cf28 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 10 Aug 2023 21:10:29 +0800 Subject: [PATCH 09/27] KubeOS: modify log level and content Signed-off-by: Yuhang Wei --- cmd/agent/server/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index 78dfd017..c474f59c 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -57,7 +57,7 @@ func (k KernelSysctl) SetConfig(config *agent.SysConfig) error { } logrus.Infof("Configured kernel.sysctl %s=%s", key, keyInfo.Value) } else { - logrus.Errorf("Failed to parse kernel.sysctl config operation %s value %s", keyInfo.Operation, keyInfo.Value) + logrus.Warnf("Failed to parse kernel.sysctl key %s value %s operation %s", key, keyInfo.Value, keyInfo.Operation) } } return nil -- Gitee From dcfb6e0397e649631ec42dcf51c3e98e914f8269 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Fri, 11 Aug 2023 19:33:56 +0800 Subject: [PATCH 10/27] KubeOS: fix updating key to kv when configuring kernel boot parameters, it should be able to update key to kv Signed-off-by: Yuhang Wei --- cmd/agent/server/config.go | 37 ++++++++++++++++++++++----------- cmd/agent/server/config_test.go | 15 +++++++------ 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index c474f59c..20af2673 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -57,7 +57,7 @@ func (k KernelSysctl) SetConfig(config *agent.SysConfig) error { } logrus.Infof("Configured kernel.sysctl %s=%s", key, keyInfo.Value) } else { - logrus.Warnf("Failed to parse kernel.sysctl key %s value %s operation %s", key, keyInfo.Value, keyInfo.Operation) + logrus.Warnf("Failed to parse kernel.sysctl key: %s value: %s operation: %s", key, keyInfo.Value, keyInfo.Operation) } } return nil @@ -317,7 +317,7 @@ func createConfigPath(configPath string) error { if err != nil { return err } - defer f.Close() + f.Close() return nil } @@ -335,11 +335,16 @@ func getGrubCfgPath() string { // handleDeleteKey deletes key if oldValue==newValue and returns "" string. Otherwier, it returns key=oldValue func handleDeleteKey(config []string, configInfo *agent.KeyInfo) string { - if len(config) == onlyKey { - logrus.Infoln("delete configuration ", config[0]) + key := config[0] + if len(config) == onlyKey && configInfo.Value == "" { + logrus.Infoln("delete configuration ", key) return "" + } else if len(config) == onlyKey && configInfo.Value != "" { + logrus.Warnf("Failed to delete key %s with inconsistent values "+ + "nil and %s", key, configInfo.Value) + return key } - key, oldValue := config[0], config[1] + oldValue := config[1] if oldValue != configInfo.Value { logrus.Warnf("Failed to delete key %s with inconsistent values "+ "%s and %s", key, oldValue, configInfo.Value) @@ -351,22 +356,30 @@ func handleDeleteKey(config []string, configInfo *agent.KeyInfo) string { // handleUpdateKey updates key if key is found, otherwise it returns old config. func handleUpdateKey(config []string, configInfo *agent.KeyInfo, isFound bool) string { - if len(config) == onlyKey { - return config[0] + key := config[0] + if !isFound && len(config) == onlyKey { + return key } - key, oldValue := config[0], config[1] - if !isFound { - return key + "=" + oldValue + if !isFound && len(config) == kvPair { + return key + "=" + config[1] } if configInfo.Operation != "" { logrus.Warnf("Unknown operation %s, updating key %s with value %s by default", configInfo.Operation, key, configInfo.Value) } + if len(config) == onlyKey && configInfo.Value == "" { + return key + } + newValue := strings.TrimSpace(configInfo.Value) + if len(config) == onlyKey && configInfo.Value != "" { + logrus.Infof("update configuration %s=%s", key, newValue) + return key + "=" + newValue + } + oldValue := config[1] if configInfo.Value == "" { logrus.Warnf("Failed to update key %s with null value", key) return key + "=" + oldValue } - newValue := strings.TrimSpace(configInfo.Value) logrus.Infof("update configuration %s=%s", key, newValue) return key + "=" + newValue } @@ -388,7 +401,7 @@ func handleAddKey(m map[string]*agent.KeyInfo, isOnlyKeyValid bool) []string { } k, v := strings.TrimSpace(key), strings.TrimSpace(keyInfo.Value) if keyInfo.Value == "" && isOnlyKeyValid { - logrus.Infoln("add configuration ", k) + logrus.Infoln("add configuration", k) configs = append(configs, k) } else if keyInfo.Value == "" { logrus.Warnf("Failed to add key %s with null value", k) diff --git a/cmd/agent/server/config_test.go b/cmd/agent/server/config_test.go index 6424885b..08daf995 100644 --- a/cmd/agent/server/config_test.go +++ b/cmd/agent/server/config_test.go @@ -262,10 +262,11 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri "": {Value: "test"}, // warning, skip, failed to add kv with empty key "selinux": {Value: "1", Operation: "delete"}, // failed to delete inconsistent kv "acpi": {Value: "off", Operation: "delete"}, // failed to delete inexistent kv + "ro": {Value: "1"}, // update key to kv }, }, }, - pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debug\spci=nomis|pci=nomis\sdebug)$`, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro=1\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debug\spci=nomis|pci=nomis\sdebug)$`, wantErr: false, }, { @@ -274,14 +275,15 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri args: args{ config: &agent.SysConfig{ Contents: map[string]*agent.KeyInfo{ - "debug": {Operation: "delete"}, // delete key - "pci": {Value: "nomis", Operation: "delete"}, // delete kv - "debugpat": {Value: "", Operation: "add"}, // passed key, operation is invalid, default to add key - "audit": {Value: "1", Operation: "add"}, // passed kv, key is inexistent, operation is invalid, default to add kv + "debug": {Operation: "delete"}, // delete key + "pci": {Value: "nomis", Operation: "delete"}, // delete kv + "debugpat": {Value: "", Operation: "add"}, // passed key, operation is invalid, default to add key + "audit": {Value: "1", Operation: "add"}, // passed kv, key is inexistent, operation is invalid, default to add kv + "nomodeset": {Value: "1", Operation: "delete"}, // delete key with inconsistent value }, }, }, - pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debugpat\saudit=1|audit=1\sdebugpat)$`, + pattern: `(?m)^\s+linux\s+\/boot\/vmlinuz\s+root=UUID=[0-1]\s+ro=1\s+rootfstype=ext4\s+nomodeset\s+oops=panic\s+softlockup_panic=0\s+nmi_watchdog=1\s+rd\.shell=0\s+selinux=0\s+crashkernel=256M\s+panic=5\s+(debugpat\saudit=1|audit=1\sdebugpat)$`, wantErr: false, }, { @@ -300,6 +302,7 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri "": {Value: "test"}, // warning, skip, failed to add kv with empty key "selinux": {Value: "1", Operation: "delete"}, "acpi": {Value: "off", Operation: "delete"}, + "ro": {Value: ""}, }, }, }, -- Gitee From 657e3e5c9bcc0fac3a079f51842d59a9e6b5e163 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 14 Aug 2023 09:53:16 +0800 Subject: [PATCH 11/27] KubeOS: fix proxy requeue bug fix the bug that proxy may work after resuming normal from error status Signed-off-by: Yuhang Wei --- cmd/proxy/controllers/os_controller.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index 09d61c0d..30f98396 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -94,7 +94,7 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re if err = r.refreshNode(ctx, &node, osInstance, osCr.Spec.SysConfigs.Version, values.SysConfigName); err != nil { return values.RequeueNow, err } - return values.RequeueNow, nil + return values.Requeue, nil } if configOps == values.UpdateConfig { osInstance.Spec.SysConfigs = osCr.Spec.SysConfigs @@ -124,7 +124,7 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re values.UpgradeConfigName); err != nil { return values.RequeueNow, err } - return values.RequeueNow, nil + return values.Requeue, nil } if err := r.setConfig(ctx, osInstance, values.UpgradeConfigName); err != nil { return values.RequeueNow, err -- Gitee From a87666e17d729feb67f987fa7038ef83b3fa2e1b Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 14 Aug 2023 16:46:06 +0800 Subject: [PATCH 12/27] KubeOS: fix operator bug of missing deep copy Fixed bug where operator didn't perform deep copy add operator ut, testing in multiple nodes env Signed-off-by: Yuhang Wei --- cmd/operator/controllers/os_controller.go | 44 +- .../controllers/os_controller_test.go | 378 ++++++++++++++---- 2 files changed, 330 insertions(+), 92 deletions(-) diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index 620739be..d36f15d4 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -15,6 +15,8 @@ package controllers import ( "context" + "encoding/json" + "fmt" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -188,7 +190,9 @@ func upgradeNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1. } continue } - updateNodeAndOSins(ctx, r, os, &node, &osInstance) + if err := updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + continue + } count++ } } @@ -196,12 +200,16 @@ func upgradeNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1. } func updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - node *corev1.Node, osInstance *upgradev1.OSInstance) { + node *corev1.Node, osInstance *upgradev1.OSInstance) error { if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { - osInstance.Spec.UpgradeConfigs = os.Spec.UpgradeConfigs + if err := deepCopySpecConfigs(os, osInstance, values.UpgradeConfigName); err != nil { + return err + } } if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { - osInstance.Spec.SysConfigs = os.Spec.SysConfigs + if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { + return err + } // exchange "grub.cmdline.current" and "grub.cmdline.next" for i, config := range osInstance.Spec.SysConfigs.Configs { if config.Model == "grub.cmdline.current" { @@ -215,11 +223,14 @@ func updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgr osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() if err := r.Update(ctx, osInstance); err != nil { log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err } node.Labels[values.LabelUpgrading] = "" if err := r.Update(ctx, node); err != nil { log.Error(err, "unable to label", "node", node.Name) + return err } + return nil } func assignConfig(ctx context.Context, r common.ReadStatusWriter, sysConfigs upgradev1.SysConfigs, @@ -307,3 +318,28 @@ func min(a, b int) int { } return b } + +func deepCopySpecConfigs(os *upgradev1.OS, osinstance *upgradev1.OSInstance, configType string) error { + switch configType { + case values.UpgradeConfigName: + data, err := json.Marshal(os.Spec.UpgradeConfigs) + if err != nil { + return err + } + if err = json.Unmarshal(data, &osinstance.Spec.UpgradeConfigs); err != nil { + return err + } + case values.SysConfigName: + data, err := json.Marshal(os.Spec.SysConfigs) + if err != nil { + return err + } + if err = json.Unmarshal(data, &osinstance.Spec.SysConfigs); err != nil { + return err + } + default: + log.Error(nil, "configType "+configType+" cannot be recognized") + return fmt.Errorf("configType %s cannot be recognized", configType) + } + return nil +} diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index 98de6d02..e59ce7e5 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -17,19 +17,12 @@ import ( "testing" "time" - "github.com/agiledragon/gomonkey/v2" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/kubernetes/scheme" - "k8s.io/client-go/util/workqueue" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/event" upgradev1 "openeuler.org/KubeOS/api/v1alpha1" "openeuler.org/KubeOS/pkg/values" @@ -37,8 +30,7 @@ import ( var _ = Describe("OsController", func() { const ( - OSName = "test-os" - + OSName = "test-os" timeout = time.Second * 20 interval = time.Millisecond * 500 ) @@ -72,26 +64,33 @@ var _ = Describe("OsController", func() { }) AfterEach(func() { - desiredTestNamespace := &v1.Namespace{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Namespace", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: testNamespace, - }, + // delete all nodes + nodeList := &v1.NodeList{} + err := k8sClient.List(context.Background(), nodeList) + Expect(err).ToNot(HaveOccurred()) + for _, node := range nodeList.Items { + k8sClient.Delete(context.Background(), &node) } - // Add any teardown steps that needs to be executed after each test - err := k8sClient.Delete(context.Background(), desiredTestNamespace, - client.PropagationPolicy(metav1.DeletePropagationForeground)) + nodeList = &v1.NodeList{} + Eventually(func() bool { + err = k8sClient.List(context.Background(), nodeList) + if err != nil || len(nodeList.Items) != 0 { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + // delete all OS CRs + osList := &upgradev1.OSList{} + err = k8sClient.List(context.Background(), osList) Expect(err).ToNot(HaveOccurred()) - - existingNamespace := &v1.Namespace{} + for _, os := range osList.Items { + k8sClient.Delete(context.Background(), &os) + } + osList = &upgradev1.OSList{} Eventually(func() bool { - err := k8sClient.Get(context.Background(), types.NamespacedName{Name: testNamespace}, - existingNamespace) - if err != nil && errors.IsNotFound(err) { + err = k8sClient.List(context.Background(), osList) + if err != nil || len(osList.Items) != 0 { return false } return true @@ -102,7 +101,7 @@ var _ = Describe("OsController", func() { It("Should label the osinstance's nodestatus to upgrading", func() { ctx := context.Background() - // create Node + // create Node1 node1Name = "test-node-" + uuid.New().String() node1 := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -131,7 +130,7 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) - // create OSInstance + // create OSInstance1 OSIns := &upgradev1.OSInstance{ TypeMeta: metav1.TypeMeta{ Kind: "OSInstance", @@ -163,6 +162,67 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v2", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + // create OS CR OS := &upgradev1.OS{ TypeMeta: metav1.TypeMeta{ @@ -206,13 +266,20 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + + createdOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) }) }) Context("When we want to configure node", func() { It("Should update OSInstance spec and update NodeStatus to config", func() { ctx := context.Background() - // create Node + // create Node1 node1Name = "test-node-" + uuid.New().String() node1 := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -228,7 +295,7 @@ var _ = Describe("OsController", func() { }, Status: v1.NodeStatus{ NodeInfo: v1.NodeSystemInfo{ - OSImage: "KubeOS v2", + OSImage: "KubeOS v1", }, }, } @@ -241,6 +308,7 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) + // create OSInstance1 OSIns := &upgradev1.OSInstance{ TypeMeta: metav1.TypeMeta{ Kind: "OSInstance", @@ -249,6 +317,9 @@ var _ = Describe("OsController", func() { ObjectMeta: metav1.ObjectMeta{ Name: node1Name, Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, }, Spec: upgradev1.OSInstanceSpec{ SysConfigs: upgradev1.SysConfigs{ @@ -261,14 +332,76 @@ var _ = Describe("OsController", func() { } Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) - osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns := &upgradev1.OSInstance{} Eventually(func() bool { - err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) return err == nil }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + OS := &upgradev1.OS{ TypeMeta: metav1.TypeMeta{ APIVersion: "upgrade.openeuler.org/v1alpha1", @@ -311,13 +444,21 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished - configedOSIns := &upgradev1.OSInstance{} + configedOSIns1 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, configedOSIns1) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(configedOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) + Expect(configedOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + + configedOSIns2 := &upgradev1.OSInstance{} Eventually(func() bool { - err := k8sClient.Get(ctx, osInsCRLookupKey, configedOSIns) + err := k8sClient.Get(ctx, osInsCRLookupKey2, configedOSIns2) return err == nil }, timeout, interval).Should(BeTrue()) - Expect(configedOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) - Expect(configedOSIns.Spec.SysConfigs.Version).Should(Equal("v2")) + Expect(configedOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) + Expect(configedOSIns2.Spec.SysConfigs.Version).Should(Equal("v2")) }) }) @@ -394,6 +535,9 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) _, ok := existingNode.Labels[values.LabelUpgrading] Expect(ok).Should(Equal(false)) + + createdOS.Spec.OpsType = "test" + Expect(k8sClient.Update(ctx, createdOS)).Should(Succeed()) }) }) @@ -401,7 +545,7 @@ var _ = Describe("OsController", func() { It("Should exchange .current and .next", func() { ctx := context.Background() - // create Node + // create Node1 node1Name = "test-node-" + uuid.New().String() node1 := &v1.Node{ ObjectMeta: metav1.ObjectMeta{ @@ -430,7 +574,7 @@ var _ = Describe("OsController", func() { return err == nil }, timeout, interval).Should(BeTrue()) - // create OSInstance + // create OSInstance1 OSIns := &upgradev1.OSInstance{ TypeMeta: metav1.TypeMeta{ Kind: "OSInstance", @@ -454,14 +598,75 @@ var _ = Describe("OsController", func() { Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) // Check that the corresponding OSIns CR has been created - osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns := &upgradev1.OSInstance{} Eventually(func() bool { - err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) return err == nil }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + // create OS CR OS := &upgradev1.OS{ TypeMeta: metav1.TypeMeta{ @@ -486,7 +691,13 @@ var _ = Describe("OsController", func() { {Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}}, }, }, - UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + UpgradeConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + {Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}}, + {Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}}, + }, + }, }, } Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) @@ -501,75 +712,66 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished - osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} + // check node1 osinstance createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { - err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) return err == nil }, timeout, interval).Should(BeTrue()) Expect(createdOSIns.Spec.SysConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) Expect(createdOSIns.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) + Expect(createdOSIns.Spec.UpgradeConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + + // check node2 osinstance + createdOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + Expect(createdOSIns2.Spec.SysConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + Expect(createdOSIns2.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) + Expect(createdOSIns2.Spec.UpgradeConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + + // check os cr spec + osCRLookupKey = types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS = &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.SysConfigs.Configs[0]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.current", Contents: []upgradev1.Content{{Key: "a", Value: "1"}}})) + Expect(createdOS.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) }) }) }) -func TestOSReconciler_DeleteOSInstance(t *testing.T) { - type fields struct { - Scheme *runtime.Scheme - Client client.Client - } - kClient, _ := client.New(cfg, client.Options{Scheme: scheme.Scheme}) +func Test_deepCopySpecConfigs(t *testing.T) { type args struct { - e event.DeleteEvent - q workqueue.RateLimitingInterface + os *upgradev1.OS + osinstance *upgradev1.OSInstance + configType string } tests := []struct { - name string - fields fields - args args + name string + args args + wantErr bool }{ { - name: "delete osinstance", - fields: fields{ - Scheme: nil, - Client: kClient, - }, + name: "error", args: args{ - e: event.DeleteEvent{ - Object: &upgradev1.OSInstance{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node1", - Namespace: "test", - }, - }, - }, - q: nil, - }, + os: &upgradev1.OS{}, + osinstance: &upgradev1.OSInstance{}, + configType: "test"}, + wantErr: true, }, } - var patchList *gomonkey.Patches - var patchDelete *gomonkey.Patches for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - r := &OSReconciler{ - Scheme: tt.fields.Scheme, - Client: tt.fields.Client, + if err := deepCopySpecConfigs(tt.args.os, tt.args.osinstance, tt.args.configType); (err != nil) != tt.wantErr { + t.Errorf("deepCopySpecConfigs() error = %v, wantErr %v", err, tt.wantErr) } - patchList = gomonkey.ApplyMethodFunc(r.Client, "List", func(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { - list.(*upgradev1.OSInstanceList).Items = []upgradev1.OSInstance{ - { - ObjectMeta: metav1.ObjectMeta{ - Name: "test-node1", - Namespace: "test", - }, - }, - } - return nil - }) - patchDelete = gomonkey.ApplyMethodReturn(r.Client, "Delete", nil) - r.DeleteOSInstance(tt.args.e, tt.args.q) }) } - defer patchDelete.Reset() - defer patchList.Reset() } -- Gitee From 9bd13d2a1b8b7282d7247caf9ba54f11bb297cd5 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Fri, 11 Aug 2023 20:25:46 +0800 Subject: [PATCH 13/27] KubeOS: add unit test add containerd_image and disk_image unit test modify make test command for only testing server and controllers Signed-off-by: Yuhang Wei --- Makefile | 2 +- cmd/agent/server/containerd_image_test.go | 10 ++- cmd/agent/server/disk_image_test.go | 83 +++++++++++++++++++---- 3 files changed, 80 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index b5b61613..eddf9e6d 100644 --- a/Makefile +++ b/Makefile @@ -133,7 +133,7 @@ kustomize: $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) ARCH := $(shell uname -m) -TEST_CMD := go test ./... -race -count=1 -timeout=300s -cover -gcflags=all=-l -p 1 +TEST_CMD := go test `go list ./cmd/... | grep -E 'server|controllers'` -race -count=1 -timeout=300s -cover -gcflags=all=-l -p 1 ifeq ($(ARCH), aarch64) TEST_CMD := ETCD_UNSUPPORTED_ARCH=arm64 $(TEST_CMD) diff --git a/cmd/agent/server/containerd_image_test.go b/cmd/agent/server/containerd_image_test.go index d7133c37..85347c83 100644 --- a/cmd/agent/server/containerd_image_test.go +++ b/cmd/agent/server/containerd_image_test.go @@ -32,7 +32,6 @@ func Test_conImageHandler_downloadImage(t *testing.T) { want string wantErr bool }{ - { name: "pullImageError", c: conImageHandler{}, @@ -62,6 +61,15 @@ func Test_conImageHandler_downloadImage(t *testing.T) { want: "update-test1/upadte.img", wantErr: false, }, + { + name: "invalid image name", + c: conImageHandler{}, + args: args{ + req: &pb.UpdateRequest{ContainerImage: "nginx;v1"}, + }, + want: "", + wantErr: true, + }, } patchPrepareEnv := gomonkey.ApplyFunc(prepareEnv, func() (preparePath, error) { return preparePath{updatePath: "update-test1/", diff --git a/cmd/agent/server/disk_image_test.go b/cmd/agent/server/disk_image_test.go index 71c5de7f..265b323d 100644 --- a/cmd/agent/server/disk_image_test.go +++ b/cmd/agent/server/disk_image_test.go @@ -21,6 +21,7 @@ import ( "crypto/x509/pkix" "encoding/hex" "encoding/pem" + "fmt" "io" "math/big" "net/http" @@ -52,36 +53,78 @@ func Test_download(t *testing.T) { want string wantErr bool }{ - // {name: "errornil", args: args{&pb.UpdateRequest{Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, - // {name: "normal", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "/persist/update.img", wantErr: false}, - // {name: "errornodir", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + {name: "errornil", args: args{&pb.UpdateRequest{Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, + {name: "error response", args: args{&pb.UpdateRequest{ImageUrl: "http://www.openeuler.abc", FlagSafe: true, Certs: &pb.CertsInfo{}}}, want: "", wantErr: true}, { name: "normal", args: args{ req: &pb.UpdateRequest{ ImageUrl: "http://www.openeuler.org/zh/", + FlagSafe: true, + Certs: &pb.CertsInfo{}, }, }, want: tmpFileForDownload, wantErr: false, }, + { + name: "disk space not enough", + args: args{ + req: &pb.UpdateRequest{ + ImageUrl: "http://www.openeuler.org/zh/", + FlagSafe: true, + Certs: &pb.CertsInfo{}, + }, + }, + want: "", + wantErr: true, + }, } - patchStatfs := gomonkey.ApplyFunc(syscall.Statfs, func(path string, stat *syscall.Statfs_t) error { + var patchStatfs *gomonkey.Patches + patchStatfs = gomonkey.ApplyFunc(syscall.Statfs, func(path string, stat *syscall.Statfs_t) error { stat.Bfree = 3000 stat.Bsize = 4096 return nil }) defer patchStatfs.Reset() - patchGetImageUrl := gomonkey.ApplyFuncReturn(getImageURL, &http.Response{ - StatusCode: http.StatusOK, - ContentLength: 5, - Body: io.NopCloser(strings.NewReader("hello")), - }, nil) + patchGetImageUrl := gomonkey.ApplyFuncSeq(getImageURL, + []gomonkey.OutputCell{ + {Values: gomonkey.Params{&http.Response{}, fmt.Errorf("error")}}, + {Values: gomonkey.Params{&http.Response{StatusCode: http.StatusBadRequest, Body: io.NopCloser(strings.NewReader(""))}, nil}}, + { + Values: gomonkey.Params{ + &http.Response{ + StatusCode: http.StatusOK, + ContentLength: 5, + Body: io.NopCloser(strings.NewReader("hello")), + }, + nil, + }, + }, + { + Values: gomonkey.Params{ + &http.Response{ + StatusCode: http.StatusOK, + ContentLength: 5, + Body: io.NopCloser(strings.NewReader("hello")), + }, + nil, + }, + }, + }, + ) defer patchGetImageUrl.Reset() patchOSCreate := gomonkey.ApplyFuncReturn(os.Create, tmpFile, nil) defer patchOSCreate.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + if tt.name == "disk space not enough" { + patchStatfs = gomonkey.ApplyFunc(syscall.Statfs, func(path string, stat *syscall.Statfs_t) error { + stat.Bfree = 1 + stat.Bsize = 4096 + return nil + }) + } got, err := download(tt.args.req) if (err != nil) != tt.wantErr { t.Errorf("download() error = %v, wantErr %v", err, tt.wantErr) @@ -160,13 +203,27 @@ func Test_getImageURL(t *testing.T) { MTLS: false, Certs: &pb.CertsInfo{}, }}, want: &http.Response{StatusCode: http.StatusOK}, wantErr: false}, + {name: "httpsLoadCertsError", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", + FlagSafe: true, + MTLS: false, + Certs: &pb.CertsInfo{}, + }}, want: &http.Response{}, wantErr: true}, + {name: "httpsMLTSLoadCertsError", args: args{req: &pb.UpdateRequest{ + ImageUrl: "https://www.openeuler.abc/zh/", + FlagSafe: true, + MTLS: true, + Certs: &pb.CertsInfo{}, + }}, want: &http.Response{}, wantErr: true}, } - patchLoadClientCerts := gomonkey.ApplyFunc(loadClientCerts, func(caCert, clientCert, clientKey string) (*http.Client, error) { - return &http.Client{}, nil + patchLoadClientCerts := gomonkey.ApplyFuncSeq(loadClientCerts, []gomonkey.OutputCell{ + {Values: gomonkey.Params{&http.Client{}, nil}}, + {Values: gomonkey.Params{&http.Client{}, fmt.Errorf("error")}}, }) defer patchLoadClientCerts.Reset() - patchLoadCaCerts := gomonkey.ApplyFunc(loadCaCerts, func(caCert string) (*http.Client, error) { - return &http.Client{}, nil + patchLoadCaCerts := gomonkey.ApplyFuncSeq(loadCaCerts, []gomonkey.OutputCell{ + {Values: gomonkey.Params{&http.Client{}, nil}}, + {Values: gomonkey.Params{&http.Client{}, fmt.Errorf("error")}}, }) defer patchLoadCaCerts.Reset() patchGet := gomonkey.ApplyFunc(http.Get, func(url string) (resp *http.Response, err error) { -- Gitee From 1bc41e7e58e99e23cd8be522cc1c2d30090975d2 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 15 Aug 2023 16:33:11 +0800 Subject: [PATCH 14/27] KubeOS: fix clean space problems add clean space function in server change docker rm position in docker_image Signed-off-by: Yuhang Wei --- cmd/agent/main.go | 4 +--- cmd/agent/server/docker_image.go | 6 +++--- cmd/agent/server/server.go | 9 +++++++++ cmd/agent/server/utils.go | 27 +++++++++++++-------------- 4 files changed, 26 insertions(+), 20 deletions(-) diff --git a/cmd/agent/main.go b/cmd/agent/main.go index 67c7e2da..7b1ed3da 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -13,8 +13,6 @@ package main import ( - "fmt" - "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -24,7 +22,7 @@ import ( ) func main() { - fmt.Println("Version is:", version.Version) + logrus.Infoln("Version is:", version.Version) l, err := server.NewListener(server.SockDir, server.SockName) if err != nil { logrus.Errorln("listen error" + err.Error()) diff --git a/cmd/agent/server/docker_image.go b/cmd/agent/server/docker_image.go index 0b6ee35b..16bcea54 100644 --- a/cmd/agent/server/docker_image.go +++ b/cmd/agent/server/docker_image.go @@ -61,13 +61,13 @@ func (d dockerImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath p if err != nil { return "", err } - if err := runCommand("docker", "cp", containerId+":/"+neededPath.rootfsFile, neededPath.updatePath); err != nil { - return "", err - } defer func() { if err := runCommand("docker", "rm", containerId); err != nil { logrus.Errorln("remove kubeos-temp container error", err) } }() + if err := runCommand("docker", "cp", containerId+":/"+neededPath.rootfsFile, neededPath.updatePath); err != nil { + return "", err + } return neededPath.tarPath, nil } diff --git a/cmd/agent/server/server.go b/cmd/agent/server/server.go index 8ac6ffd6..f8cbb410 100644 --- a/cmd/agent/server/server.go +++ b/cmd/agent/server/server.go @@ -112,6 +112,15 @@ func (s *Server) update(req *pb.UpdateRequest) error { return fmt.Errorf("image type %s cannot be recognized", action) } imagePath, err := handler.downloadImage(req) + defer func() { + if err != nil { + path := newPreparePath() + if err := cleanSpace(path.updatePath, path.mountPath, path.imagePath); err != nil { + logrus.Errorln("clean space error " + err.Error()) + } + logrus.Infoln("clean space success") + } + }() if err != nil { return err } diff --git a/cmd/agent/server/utils.go b/cmd/agent/server/utils.go index b42db18f..d2d09463 100644 --- a/cmd/agent/server/utils.go +++ b/cmd/agent/server/utils.go @@ -190,26 +190,25 @@ func prepareEnv() (preparePath, error) { if err := checkDiskSize(needGBSize, PersistDir); err != nil { return preparePath{}, err } - rootfsFile := rootfsArchive - updatePath := splicePath(PersistDir, updateDir) - mountPath := splicePath(updatePath, mountDir) - tarPath := splicePath(updatePath, rootfsFile) - imagePath := splicePath(PersistDir, osImageName) - - if err := cleanSpace(updatePath, mountPath, imagePath); err != nil { + upgradePath := newPreparePath() + if err := cleanSpace(upgradePath.updatePath, upgradePath.mountPath, upgradePath.imagePath); err != nil { return preparePath{}, err } - if err := os.MkdirAll(mountPath, imgPermission); err != nil { + if err := os.MkdirAll(upgradePath.mountPath, imgPermission); err != nil { return preparePath{}, err } - upgradePath := preparePath{ + return upgradePath, nil +} + +func newPreparePath() preparePath { + updatePath := splicePath(PersistDir, updateDir) + return preparePath{ updatePath: updatePath, - mountPath: mountPath, - tarPath: tarPath, - imagePath: imagePath, - rootfsFile: rootfsFile, + mountPath: splicePath(updatePath, mountDir), + tarPath: splicePath(updatePath, rootfsArchive), + imagePath: splicePath(PersistDir, osImageName), + rootfsFile: rootfsArchive, } - return upgradePath, nil } func checkDiskSize(needGBSize int, path string) error { -- Gitee From 9454d10ab546b0edd50d1e0d68b386d00c95fb4b Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 15 Aug 2023 16:24:25 +0800 Subject: [PATCH 15/27] KubeOS: add line breaks Signed-off-by: Yuhang Wei --- scripts/admin-container/set-ssh-pub-key.service | 2 +- scripts/admin-container/set-ssh-pub-key.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/admin-container/set-ssh-pub-key.service b/scripts/admin-container/set-ssh-pub-key.service index cf214069..84dd12d9 100644 --- a/scripts/admin-container/set-ssh-pub-key.service +++ b/scripts/admin-container/set-ssh-pub-key.service @@ -12,4 +12,4 @@ Description="set ssh authorized keys according to the secret which is set by user" [Service] -ExecStart="/usr/local/bin/set-ssh-pub-key.sh" \ No newline at end of file +ExecStart="/usr/local/bin/set-ssh-pub-key.sh" diff --git a/scripts/admin-container/set-ssh-pub-key.sh b/scripts/admin-container/set-ssh-pub-key.sh index aa706c2d..e91a15de 100755 --- a/scripts/admin-container/set-ssh-pub-key.sh +++ b/scripts/admin-container/set-ssh-pub-key.sh @@ -23,4 +23,4 @@ if [ ! -f "$authorized_file" ]; then chmod 600 "$authorized_file" fi -echo "$ssh_pub" >> "$authorized_file" \ No newline at end of file +echo "$ssh_pub" >> "$authorized_file" -- Gitee From 9f0f03251a1bd0449f4e7e7ab5699e4771fbaa56 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Fri, 18 Aug 2023 17:19:53 +0800 Subject: [PATCH 16/27] KubeOS: update version Signed-off-by: Yuhang Wei --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 7dea76ed..ee90284c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.0.1 +1.0.4 -- Gitee From a01e5ebfd9c314b840eb3a652cda71e33954a403 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Wed, 16 Aug 2023 22:41:44 +0800 Subject: [PATCH 17/27] KubeOS: modify code for clean code Signed-off-by: Yuhang Wei --- cmd/agent/server/config.go | 4 ++++ cmd/operator/controllers/os_controller.go | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index 20af2673..a96370da 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -317,6 +317,10 @@ func createConfigPath(configPath string) error { if err != nil { return err } + err = f.Chmod(defaultKernelConPermission) + if err != nil { + return err + } f.Close() return nil } diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index 620739be..e152681b 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -94,7 +94,11 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) func (r *OSReconciler) SetupWithManager(mgr ctrl.Manager) error { if err := mgr.GetFieldIndexer().IndexField(context.Background(), &upgradev1.OSInstance{}, values.OsiStatusName, func(rawObj client.Object) []string { - osi := rawObj.(*upgradev1.OSInstance) + osi, ok := rawObj.(*upgradev1.OSInstance) + if !ok { + log.Error(nil, "failed to convert to osInstance") + return []string{} + } return []string{osi.Spec.NodeStatus} }); err != nil { return err -- Gitee From 0c0f3c7ef5749b8806511d9e3312f6cacd4cca04 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Tue, 15 Aug 2023 12:29:11 +0000 Subject: [PATCH 18/27] KubeOS: fix the issue that osinstance is not updated in time during proxy upgrade. Fix the problem that osinstance is not updated in time during proxy upgrade, but the node upgrade tag has been added. As a result, the configuration is skipped and the upgrade is directly performed. Signed-off-by: liyuanr --- cmd/proxy/controllers/os_controller.go | 11 ++ cmd/proxy/controllers/os_controller_test.go | 126 ++++++++++++++++++++ 2 files changed, 137 insertions(+) diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index 30f98396..b8d0f802 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -126,6 +126,17 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re } return values.Requeue, nil } + if _, ok := node.Labels[values.LabelUpgrading]; ok && + osInstance.Spec.NodeStatus == values.NodeStatusIdle.String() { + log.Info("node has upgrade label, but osInstance.spec.nodestaus idle ", + "operation:", "refesh node and wait opetaot reassgin") + if err = r.refreshNode(ctx, &node, osInstance, osCr.Spec.UpgradeConfigs.Version, + values.UpgradeConfigName); err != nil { + return values.RequeueNow, err + } + return values.Requeue, nil + } + if err := r.setConfig(ctx, osInstance, values.UpgradeConfigName); err != nil { return values.RequeueNow, err } diff --git a/cmd/proxy/controllers/os_controller_test.go b/cmd/proxy/controllers/os_controller_test.go index d63f176e..27cb0cd6 100644 --- a/cmd/proxy/controllers/os_controller_test.go +++ b/cmd/proxy/controllers/os_controller_test.go @@ -1098,4 +1098,130 @@ var _ = Describe("OsController", func() { Expect(ok).Should(Equal(false)) }) }) + + Context("When node has upgrade label but osinstance.spec.nodestatus is idle", func() { + It("Should be able to refresh node and wait operator reassgin upgrade", func() { + ctx := context.Background() + By("Creating a worker node") + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + values.LabelUpgrading: "", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v2", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + reconciler.hostName = node1Name + + By("Creating the corresponding OSInstance") + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + NodeStatus: values.NodeStatusIdle.String(), + }, + Status: upgradev1.OSInstanceStatus{}, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + By("Creating a OS custom resource") + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "upgrade", + MaxUnavailable: 3, + OSVersion: "KubeOS v2", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "c"}, + {Key: "key2", Value: "d"}, + }, + }, + }, + }, + UpgradeConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) + + time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(false)) + }) + }) }) -- Gitee From c4c7f94101ecd6c57cf48f1e526f585fb44cd213 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 21 Aug 2023 09:58:45 +0800 Subject: [PATCH 19/27] KubeOS: update config log contents Signed-off-by: Yuhang Wei --- cmd/agent/server/config.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index a96370da..f186ee6e 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -57,7 +57,7 @@ func (k KernelSysctl) SetConfig(config *agent.SysConfig) error { } logrus.Infof("Configured kernel.sysctl %s=%s", key, keyInfo.Value) } else { - logrus.Warnf("Failed to parse kernel.sysctl key: %s value: %s operation: %s", key, keyInfo.Value, keyInfo.Operation) + logrus.Warnf("Failed to parse kernel.sysctl, key: %s, value: %s, operation: %s", key, keyInfo.Value, keyInfo.Operation) } } return nil @@ -97,7 +97,11 @@ type GrubCmdline struct { // SetConfig sets grub.cmdline configuration func (g GrubCmdline) SetConfig(config *agent.SysConfig) error { - logrus.Info("start set grub.cmdline configuration") + if g.isCurPartition { + logrus.Info("start set grub.cmdline.current configuration") + } else { + logrus.Info("start set grub.cmdline.next configuration") + } fileExist, err := checkFileExist(getGrubCfgPath()) if err != nil { logrus.Errorf("Failed to find config path: %v", err) -- Gitee From 3e22e0c5f3667aa877492423d454c12cf96b9de2 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 24 Aug 2023 11:00:04 +0800 Subject: [PATCH 20/27] KubeOS: modify code for clean code add err handler in NewControllerManager Signed-off-by: Yuhang Wei --- cmd/operator/main.go | 6 +++++- cmd/proxy/main.go | 8 ++++++-- pkg/common/common.go | 7 +++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 17b74e1b..8249ad2d 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -41,7 +41,11 @@ func init() { } func main() { - mgr := common.NewControllerManager(setupLog, scheme) + mgr, err := common.NewControllerManager(setupLog, scheme) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } if err := (&controllers.OSReconciler{ Client: mgr.GetClient(), diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index ce1f58df..3a537d90 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -44,10 +44,14 @@ func init() { } func main() { - mgr := common.NewControllerManager(setupLog, scheme) + var err error + mgr, err := common.NewControllerManager(setupLog, scheme) + if err != nil { + setupLog.Error(err, "unable to start manager") + os.Exit(1) + } reconciler := controllers.NewOSReconciler(mgr) - var err error if reconciler.Connection, err = agentclient.New("unix://" + filepath.Join(server.SockDir, server.SockName)); err != nil { setupLog.Error(err, "Error running proxy") } diff --git a/pkg/common/common.go b/pkg/common/common.go index 4653a619..e888179f 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -15,7 +15,6 @@ package common import ( "flag" - "os" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" @@ -33,7 +32,7 @@ type ReadStatusWriter interface { } // NewControllerManager configure and return a manager -func NewControllerManager(setupLog logr.Logger, scheme *runtime.Scheme) manager.Manager { +func NewControllerManager(setupLog logr.Logger, scheme *runtime.Scheme) (manager.Manager, error) { opts := zap.Options{} opts.BindFlags(flag.CommandLine) @@ -46,7 +45,7 @@ func NewControllerManager(setupLog logr.Logger, scheme *runtime.Scheme) manager. }) if err != nil { setupLog.Error(err, "unable to start manager") - os.Exit(1) + return nil, err } - return mgr + return mgr, nil } -- Gitee From 3054927f5dbc3f8eff97e9e29466cc35c804110b Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Wed, 16 Aug 2023 14:27:36 +0800 Subject: [PATCH 21/27] KubeOS:add unit tests modify proxy ut to avoid panic bug add config, disk_image, operator, proxy, server and utils unit tests Signed-off-by: Yuhang Wei --- cmd/agent/server/config_test.go | 41 ++- cmd/agent/server/disk_image_test.go | 56 +++- cmd/agent/server/server_test.go | 1 - cmd/agent/server/utils_test.go | 112 +++++++ .../controllers/os_controller_test.go | 232 +++++++++++++- cmd/proxy/controllers/os_controller_test.go | 295 +++++++++++++++--- 6 files changed, 685 insertions(+), 52 deletions(-) diff --git a/cmd/agent/server/config_test.go b/cmd/agent/server/config_test.go index 08daf995..29bb9268 100644 --- a/cmd/agent/server/config_test.go +++ b/cmd/agent/server/config_test.go @@ -75,6 +75,16 @@ func TestKernelSysctl_SetConfig(t *testing.T) { }, }}, }, + { + name: "nil key", + k: KernelSysctl{}, + args: args{config: &agent.SysConfig{ + Contents: map[string]*agent.KeyInfo{ + "": {Value: "1"}, + }, + }}, + wantErr: true, + }, } tmpDir := t.TempDir() patchGetProcPath := gomonkey.ApplyFuncReturn(getDefaultProcPath, tmpDir+"/") @@ -320,8 +330,7 @@ menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestri defer patchGetConfigPartition.Reset() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - g := GrubCmdline{} - if err := g.SetConfig(tt.args.config); (err != nil) != tt.wantErr { + if err := tt.g.SetConfig(tt.args.config); (err != nil) != tt.wantErr { t.Errorf("GrubCmdline.SetConfig() error = %v, wantErr %v", err, tt.wantErr) } contents, err := os.ReadFile(grubCfgPath) @@ -513,3 +522,31 @@ func Test_ConfigFactoryTemplate(t *testing.T) { }) } } + +func Test_convertNewConfigsToString(t *testing.T) { + type args struct { + newConfigs []string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "error", args: args{newConfigs: []string{"a"}}, want: "", wantErr: true}, + } + patchFprintf := gomonkey.ApplyFuncReturn(fmt.Fprintf, 0, fmt.Errorf("error")) + defer patchFprintf.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := convertNewConfigsToString(tt.args.newConfigs) + if (err != nil) != tt.wantErr { + t.Errorf("convertNewConfigsToString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("convertNewConfigsToString() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/cmd/agent/server/disk_image_test.go b/cmd/agent/server/disk_image_test.go index 265b323d..f970bd7d 100644 --- a/cmd/agent/server/disk_image_test.go +++ b/cmd/agent/server/disk_image_test.go @@ -23,6 +23,7 @@ import ( "encoding/pem" "fmt" "io" + "io/fs" "math/big" "net/http" "os" @@ -159,6 +160,7 @@ func Test_checkSumMatch(t *testing.T) { wantErr: false, }, {name: "error", args: args{filePath: tmpFileForCheckSum, checkSum: "aaa"}, wantErr: true}, + {name: "unfound error", args: args{filePath: "", checkSum: "aaa"}, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -271,6 +273,7 @@ func Test_loadCaCerts(t *testing.T) { }, wantErr: false, }, + {name: "no cert", args: args{caCert: ""}, wantErr: true}, } patchGetCertPath := gomonkey.ApplyFuncReturn(getCertPath, "") defer patchGetCertPath.Reset() @@ -339,12 +342,22 @@ func Test_certExist(t *testing.T) { }{ {name: "fileEmpty", args: args{certFile: ""}, wantErr: true}, {name: "fileNotExist", args: args{certFile: "bb.txt"}, wantErr: true}, + {name: "unknow error", args: args{certFile: "cc.txt"}, wantErr: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + var patchStat *gomonkey.Patches + if tt.name == "unknow error" { + patchStat = gomonkey.ApplyFunc(os.Stat, func(name string) (fs.FileInfo, error) { + return fs.FileInfo(nil), fmt.Errorf("error") + }) + } if err := certExist(tt.args.certFile); (err != nil) != tt.wantErr { t.Errorf("certExist() error = %v, wantErr %v", err, tt.wantErr) } + if tt.name == "unknow error" { + patchStat.Reset() + } }) } defer os.RemoveAll("/etc/KubeOS/") @@ -396,8 +409,17 @@ func Test_diskHandler_getRootfsArchive(t *testing.T) { want: "/persist/update.img", wantErr: false, }, + { + name: "error", d: diskHandler{}, + args: args{req: &pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/"}, neededPath: preparePath{}}, + want: "", + wantErr: true, + }, } - patchDownload := gomonkey.ApplyFuncReturn(download, "/persist/update.img", nil) + patchDownload := gomonkey.ApplyFuncSeq(download, []gomonkey.OutputCell{ + {Values: gomonkey.Params{"/persist/update.img", nil}}, + {Values: gomonkey.Params{"", fmt.Errorf("error")}}, + }) defer patchDownload.Reset() patchCheckSumMatch := gomonkey.ApplyFuncReturn(checkSumMatch, nil) defer patchCheckSumMatch.Reset() @@ -415,3 +437,35 @@ func Test_diskHandler_getRootfsArchive(t *testing.T) { }) } } + +func Test_diskHandler_downloadImage(t *testing.T) { + type args struct { + req *pb.UpdateRequest + } + tests := []struct { + name string + d diskHandler + args args + want string + wantErr bool + }{ + {name: "normal", d: diskHandler{}, args: args{req: &pb.UpdateRequest{ImageUrl: "http://www.openeuler.org/zh/"}}, want: "/persist/update.img", wantErr: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := diskHandler{} + patchGetRootfsArchive := gomonkey.ApplyPrivateMethod(reflect.TypeOf(d), "getRootfsArchive", func(_ *diskHandler, _ *pb.UpdateRequest, _ preparePath) (string, error) { + return "/persist/update.img", nil + }) + got, err := d.downloadImage(tt.args.req) + if (err != nil) != tt.wantErr { + t.Errorf("diskHandler.downloadImage() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("diskHandler.downloadImage() = %v, want %v", got, tt.want) + } + patchGetRootfsArchive.Reset() + }) + } +} diff --git a/cmd/agent/server/server_test.go b/cmd/agent/server/server_test.go index 74e2eade..15b6f5ea 100644 --- a/cmd/agent/server/server_test.go +++ b/cmd/agent/server/server_test.go @@ -311,7 +311,6 @@ func TestServer_Configure(t *testing.T) { want *pb.ConfigureResponse wantErr bool }{ - // TODO: Add test cases. { name: "nil", fields: fields{UnimplementedOSServer: pb.UnimplementedOSServer{}, disableReboot: true}, diff --git a/cmd/agent/server/utils_test.go b/cmd/agent/server/utils_test.go index 0796bce4..da53c0e6 100644 --- a/cmd/agent/server/utils_test.go +++ b/cmd/agent/server/utils_test.go @@ -15,6 +15,7 @@ package server import ( "archive/tar" + "fmt" "os" "os/exec" "reflect" @@ -58,12 +59,14 @@ func Test_install(t *testing.T) { }{ {name: "normal uefi", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: false}, {name: "normal legacy", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: false}, + {name: "get boot mode error", args: args{imagePath: "aa.txt", side: "/dev/sda3", next: "A"}, wantErr: true}, } patchRunCommand := gomonkey.ApplyFuncReturn(runCommand, nil) defer patchRunCommand.Reset() patchGetBootMode := gomonkey.ApplyFuncSeq(getBootMode, []gomonkey.OutputCell{ {Values: gomonkey.Params{"uefi", nil}}, {Values: gomonkey.Params{"legacy", nil}}, + {Values: gomonkey.Params{"", fmt.Errorf("error")}}, }) defer patchGetBootMode.Reset() for _, tt := range tests { @@ -89,10 +92,12 @@ func Test_getNextPart(t *testing.T) { }{ {name: "switch to sda3", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "/dev/sda3", want1: "B", wantErr: false}, {name: "switch to sda2", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "/dev/sda2", want1: "A", wantErr: false}, + {name: "error", args: args{partA: "/dev/sda2", partB: "/dev/sda3"}, want: "", want1: "", wantErr: true}, } patchExecCommand := gomonkey.ApplyMethodSeq(&exec.Cmd{}, "CombinedOutput", []gomonkey.OutputCell{ {Values: gomonkey.Params{[]byte("/"), nil}}, {Values: gomonkey.Params{[]byte(""), nil}}, + {Values: gomonkey.Params{[]byte(""), fmt.Errorf("error")}}, }) defer patchExecCommand.Reset() for _, tt := range tests { @@ -242,10 +247,16 @@ func Test_getBootMode(t *testing.T) { want: "legacy", wantErr: false, }, + { + name: "error", + want: "", + wantErr: true, + }, } patchOSStat := gomonkey.ApplyFuncSeq(os.Stat, []gomonkey.OutputCell{ {Values: gomonkey.Params{nil, nil}}, {Values: gomonkey.Params{nil, os.ErrNotExist}}, + {Values: gomonkey.Params{nil, fmt.Errorf("fake error")}}, }) defer patchOSStat.Reset() for _, tt := range tests { @@ -326,3 +337,104 @@ func Test_checkOCIImageDigestMatch(t *testing.T) { }) } } + +func Test_runCommandWithOut(t *testing.T) { + type args struct { + name string + args []string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + {name: "error", args: args{name: "/mmm", args: []string{"", ""}}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := runCommandWithOut(tt.args.name, tt.args.args...) + if (err != nil) != tt.wantErr { + t.Errorf("runCommandWithOut() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("runCommandWithOut() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getRootfsDisks(t *testing.T) { + tests := []struct { + name string + want string + want1 string + wantErr bool + }{ + {name: "error", want: "", want1: "", wantErr: true}, + } + patchRunCommandWithOut := gomonkey.ApplyFuncSeq(runCommandWithOut, []gomonkey.OutputCell{ + {Values: gomonkey.Params{"", fmt.Errorf("fake error")}}, + }) + defer patchRunCommandWithOut.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1, err := getRootfsDisks() + if (err != nil) != tt.wantErr { + t.Errorf("getRootfsDisks() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getRootfsDisks() got = %v, want %v", got, tt.want) + } + if got1 != tt.want1 { + t.Errorf("getRootfsDisks() got1 = %v, want %v", got1, tt.want1) + } + }) + } +} + +func Test_checkDiskSize(t *testing.T) { + type args struct { + needGBSize int + path string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "zero GB need", args: args{needGBSize: 0, path: "/dev/sda"}, wantErr: false}, + {name: "disk not enough", args: args{needGBSize: 100000, path: "/dev/sda"}, wantErr: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := checkDiskSize(tt.args.needGBSize, tt.args.path); (err != nil) != tt.wantErr { + t.Errorf("checkDiskSize() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_deleteFile(t *testing.T) { + type args struct { + path string + } + tests := []struct { + name string + args args + wantErr bool + }{ + {name: "error", args: args{path: "/mmm"}, wantErr: true}, + } + patchStat := gomonkey.ApplyFuncReturn(os.Stat, nil, fmt.Errorf("fake error")) + defer patchStat.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := deleteFile(tt.args.path); (err != nil) != tt.wantErr { + t.Errorf("deleteFile() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index e59ce7e5..6cc2760e 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -14,17 +14,22 @@ package controllers import ( "context" + "fmt" + "reflect" "testing" "time" + "github.com/agiledragon/gomonkey/v2" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" - upgradev1 "openeuler.org/KubeOS/api/v1alpha1" + "openeuler.org/KubeOS/pkg/common" "openeuler.org/KubeOS/pkg/values" ) @@ -775,3 +780,228 @@ func Test_deepCopySpecConfigs(t *testing.T) { }) } } + +func Test_getConfigOSInstances(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + } + tests := []struct { + name string + args args + want []upgradev1.OSInstance + wantErr bool + }{ + { + name: "list error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + }, + want: nil, + wantErr: true, + }, + } + patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ + {Values: gomonkey.Params{fmt.Errorf("list error")}}, + }) + defer patchList.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getConfigOSInstances(tt.args.ctx, tt.args.r) + if (err != nil) != tt.wantErr { + t.Errorf("getConfigOSInstances() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getConfigOSInstances() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_checkUpgrading(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + maxUnavailable int + } + tests := []struct { + name string + args args + want int + wantErr bool + }{ + { + name: "label error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + }, + want: 0, + wantErr: true, + }, + } + patchNewRequirement := gomonkey.ApplyFuncSeq(labels.NewRequirement, []gomonkey.OutputCell{ + {Values: gomonkey.Params{nil, fmt.Errorf("label error")}}, + {Values: gomonkey.Params{nil, nil}}, + }) + defer patchNewRequirement.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := checkUpgrading(tt.args.ctx, tt.args.r, tt.args.maxUnavailable) + if (err != nil) != tt.wantErr { + t.Errorf("checkUpgrading() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("checkUpgrading() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getIdleOSInstances(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + limit int + } + tests := []struct { + name string + args args + want []upgradev1.OSInstance + wantErr bool + }{ + { + name: "list error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + limit: 1, + }, + want: nil, + wantErr: true, + }, + } + patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ + {Values: gomonkey.Params{fmt.Errorf("list error")}}, + }) + defer patchList.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getIdleOSInstances(tt.args.ctx, tt.args.r, tt.args.limit) + if (err != nil) != tt.wantErr { + t.Errorf("getIdleOSInstances() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getIdleOSInstances() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getNodes(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + limit int + reqs []labels.Requirement + } + tests := []struct { + name string + args args + want []corev1.Node + wantErr bool + }{ + { + name: "list error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + limit: 1, + }, + want: nil, + wantErr: true, + }, + } + patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ + {Values: gomonkey.Params{fmt.Errorf("list error")}}, + }) + defer patchList.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getNodes(tt.args.ctx, tt.args.r, tt.args.limit, tt.args.reqs...) + if (err != nil) != tt.wantErr { + t.Errorf("getNodes() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getNodes() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getAndUpdateOS(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + name types.NamespacedName + } + tests := []struct { + name string + args args + wantOs upgradev1.OS + wantNodeNum int + wantErr bool + }{ + { + name: "label error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + name: types.NamespacedName{Namespace: "test_ns", Name: "test"}, + }, + wantOs: upgradev1.OS{}, + wantNodeNum: 0, + wantErr: true, + }, + { + name: "get nodes error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + name: types.NamespacedName{Namespace: "test_ns", Name: "test"}, + }, + wantOs: upgradev1.OS{}, + wantNodeNum: 0, + wantErr: true, + }, + } + patchGet := gomonkey.ApplyMethodReturn(&OSReconciler{}, "Get", nil) + defer patchGet.Reset() + patchNewRequirement := gomonkey.ApplyFuncSeq(labels.NewRequirement, []gomonkey.OutputCell{ + {Values: gomonkey.Params{nil, fmt.Errorf("label error")}}, + {Values: gomonkey.Params{&labels.Requirement{}, nil}}, + }) + defer patchNewRequirement.Reset() + patchGetNodes := gomonkey.ApplyFuncReturn(getNodes, nil, fmt.Errorf("get nodes error")) + defer patchGetNodes.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotOs, gotNodeNum, err := getAndUpdateOS(tt.args.ctx, tt.args.r, tt.args.name) + if (err != nil) != tt.wantErr { + t.Errorf("getAndUpdateOS() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(gotOs, tt.wantOs) { + t.Errorf("getAndUpdateOS() gotOs = %v, want %v", gotOs, tt.wantOs) + } + if gotNodeNum != tt.wantNodeNum { + t.Errorf("getAndUpdateOS() gotNodeNum = %v, want %v", gotNodeNum, tt.wantNodeNum) + } + }) + } +} diff --git a/cmd/proxy/controllers/os_controller_test.go b/cmd/proxy/controllers/os_controller_test.go index 27cb0cd6..14b6b666 100644 --- a/cmd/proxy/controllers/os_controller_test.go +++ b/cmd/proxy/controllers/os_controller_test.go @@ -16,25 +16,27 @@ import ( "context" "fmt" "reflect" + "testing" "time" "github.com/agiledragon/gomonkey/v2" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" - + "k8s.io/kubectl/pkg/drain" upgradev1 "openeuler.org/KubeOS/api/v1alpha1" "openeuler.org/KubeOS/pkg/agentclient" + "openeuler.org/KubeOS/pkg/common" "openeuler.org/KubeOS/pkg/values" ) var _ = Describe("OsController", func() { const ( - OSName = "test-os" - + OSName = "test-os" timeout = time.Second * 20 interval = time.Millisecond * 500 ) @@ -67,6 +69,24 @@ var _ = Describe("OsController", func() { testNamespace = existingNamespace.Name }) + AfterEach(func() { + // delete all OS CRs + osList := &upgradev1.OSList{} + err := k8sClient.List(context.Background(), osList) + Expect(err).ToNot(HaveOccurred()) + for _, os := range osList.Items { + k8sClient.Delete(context.Background(), &os) + } + osList = &upgradev1.OSList{} + Eventually(func() bool { + err = k8sClient.List(context.Background(), osList) + if err != nil || len(osList.Items) != 0 { + return false + } + return true + }, timeout, interval).Should(BeTrue()) + }) + Context("When we want to rollback", func() { It("Should be able to rollback to previous version", func() { ctx := context.Background() @@ -182,6 +202,13 @@ var _ = Describe("OsController", func() { Expect(k8sClient.Status().Update(ctx, existingNode)).Should(Succeed()) By("Changing the OS Spec config to trigger reconcile") + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOSIns.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOSIns)).Should(Succeed()) createdOS = &upgradev1.OS{} Eventually(func() bool { err := k8sClient.Get(ctx, osCRLookupKey, createdOS) @@ -190,7 +217,7 @@ var _ = Describe("OsController", func() { createdOS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} Expect(k8sClient.Update(ctx, createdOS)).Should(Succeed()) - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) @@ -335,7 +362,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) By("Checking the OSInstance status config version") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -451,7 +478,7 @@ var _ = Describe("OsController", func() { OSVersion: "KubeOS v2", FlagSafe: true, MTLS: false, - EvictPodForce: true, + EvictPodForce: false, SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, UpgradeConfigs: upgradev1.SysConfigs{ Version: "v2", @@ -478,7 +505,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) By("Checking the OSInstance status config version") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -566,7 +593,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) By("Checking the existence of new OSInstance") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey := types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns := &upgradev1.OSInstance{} Eventually(func() bool { @@ -644,7 +671,7 @@ var _ = Describe("OsController", func() { }, UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, }, - Status: upgradev1.OSInstanceStatus{}, + Status: upgradev1.OSInstanceStatus{SysConfigs: upgradev1.SysConfigs{Version: "v1"}}, } Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) @@ -711,7 +738,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v1")) By("Checking the OSInstance status config version failed to be updated") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -721,10 +748,18 @@ var _ = Describe("OsController", func() { Expect(createdOSIns.Status.SysConfigs.Version).Should(Equal("v1")) Expect(createdOSIns.Spec.SysConfigs.Version).Should(Equal("v2")) - By("Changing the OS Spec config version to previous one") + By("Changing the OS and OSi Spec config version to previous one") OS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} Expect(k8sClient.Update(ctx, OS)).Should(Succeed()) - time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOSIns.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOSIns)).Should(Succeed()) + + time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) @@ -799,7 +834,6 @@ var _ = Describe("OsController", func() { }, }, }, - Status: upgradev1.OSInstanceStatus{}, } Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) @@ -870,7 +904,7 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) By("Checking the OSInstance status config version failed to be updated") - time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -889,6 +923,16 @@ var _ = Describe("OsController", func() { err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) return err == nil }, timeout, interval).Should(BeTrue()) + + osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOSIns.Spec.UpgradeConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOSIns)).Should(Succeed()) + // NodeStatus changes to idle then operator can reassign configs to this node Expect(createdOSIns.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) existingNode = &v1.Node{} @@ -906,37 +950,7 @@ var _ = Describe("OsController", func() { It("Should be able to rollback to previous config version to jump out of error state", func() { ctx := context.Background() - By("Creating a worker node") node1Name = "test-node-" + uuid.New().String() - node1 := &v1.Node{ - ObjectMeta: metav1.ObjectMeta{ - Name: node1Name, - Namespace: testNamespace, - Labels: map[string]string{ - "beta.kubernetes.io/os": "linux", - values.LabelUpgrading: "", - }, - }, - TypeMeta: metav1.TypeMeta{ - APIVersion: "v1", - Kind: "Node", - }, - Status: v1.NodeStatus{ - NodeInfo: v1.NodeSystemInfo{ - OSImage: "KubeOS v2", - }, - }, - } - err := k8sClient.Create(ctx, node1) - Expect(err).ToNot(HaveOccurred()) - existingNode := &v1.Node{} - Eventually(func() bool { - err := k8sClient.Get(context.Background(), - types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) - return err == nil - }, timeout, interval).Should(BeTrue()) - reconciler.hostName = node1Name - By("Creating the corresponding OSInstance") OSIns := &upgradev1.OSInstance{ TypeMeta: metav1.TypeMeta{ @@ -1004,6 +1018,37 @@ var _ = Describe("OsController", func() { createdOSIns.Status.SysConfigs.Version = "v1" Expect(k8sClient.Status().Update(ctx, createdOSIns)).Should(Succeed()) Expect(createdOSIns.Status.UpgradeConfigs.Version).Should(Equal("v2")) + Expect(createdOSIns.Status.SysConfigs.Version).Should(Equal("v1")) + + By("Creating a worker node") + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + values.LabelUpgrading: "", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v2", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + reconciler.hostName = node1Name // stub r.Connection.ConfigureSpec() patchConfigure := gomonkey.ApplyMethod(reflect.TypeOf(reconciler.Connection), @@ -1067,7 +1112,6 @@ var _ = Describe("OsController", func() { By("Checking the OSInstance status config version failed to be updated") time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished - osInsCRLookupKey = types.NamespacedName{Name: node1Name, Namespace: testNamespace} createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { err := k8sClient.Get(ctx, osInsCRLookupKey, createdOSIns) @@ -1077,8 +1121,21 @@ var _ = Describe("OsController", func() { Expect(createdOSIns.Spec.SysConfigs.Version).Should(Equal("v2")) By("Changing the OS Spec config version to previous one") - OS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} - Expect(k8sClient.Update(ctx, OS)).Should(Succeed()) + createdOS = &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + createdOS.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, createdOS)).Should(Succeed()) + getOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey, getOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + getOSIns.Spec.SysConfigs = upgradev1.SysConfigs{Version: "v1", Configs: []upgradev1.SysConfig{}} + Expect(k8sClient.Update(ctx, getOSIns)).Should(Succeed()) + time.Sleep(2 * time.Second) // sleep a while to make sure Reconcile finished createdOSIns = &upgradev1.OSInstance{} Eventually(func() bool { @@ -1225,3 +1282,147 @@ var _ = Describe("OsController", func() { }) }) }) + +func Test_evictNode(t *testing.T) { + type args struct { + drainer *drain.Helper + node *corev1.Node + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "node unschedulable", + args: args{ + drainer: &drain.Helper{}, + node: &corev1.Node{Spec: v1.NodeSpec{Unschedulable: true}}, + }, + wantErr: false, + }, + { + name: "runCordonError1", + args: args{ + drainer: &drain.Helper{}, + node: &corev1.Node{}, + }, + wantErr: true, + }, + { + name: "runNodeDrainError", + args: args{ + drainer: &drain.Helper{}, + node: &corev1.Node{}, + }, + wantErr: true, + }, + { + name: "runUncordonError2", + args: args{ + drainer: &drain.Helper{}, + node: &corev1.Node{}, + }, + wantErr: true, + }, + } + patchRunCordon := gomonkey.ApplyFuncSeq(drain.RunCordonOrUncordon, []gomonkey.OutputCell{ + {Values: gomonkey.Params{fmt.Errorf("cordon error")}}, + {Values: gomonkey.Params{nil}}, + {Values: gomonkey.Params{fmt.Errorf("cordon error")}}, + {Values: gomonkey.Params{nil}}, + {Values: gomonkey.Params{nil}}, + }) + defer patchRunCordon.Reset() + patchRunNodeDrain := gomonkey.ApplyFuncSeq(drain.RunNodeDrain, []gomonkey.OutputCell{ + {Values: gomonkey.Params{fmt.Errorf("node drain error")}}, + {Values: gomonkey.Params{fmt.Errorf("node drain error")}}, + }) + defer patchRunNodeDrain.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := evictNode(tt.args.drainer, tt.args.node); (err != nil) != tt.wantErr { + t.Errorf("evictNode() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_updateConfigStatus(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + osInstance *upgradev1.OSInstance + configType string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "invalid config type", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + osInstance: &upgradev1.OSInstance{}, + configType: "invalid", + }, + wantErr: true, + }, + } + patchUpdate := gomonkey.ApplyMethodReturn(&OSReconciler{}, "Update", fmt.Errorf("update error")) + patchStatus := gomonkey.ApplyMethodReturn(&OSReconciler{}, "Status", &OSReconciler{}) + defer patchUpdate.Reset() + defer patchStatus.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := updateConfigStatus(tt.args.ctx, tt.args.r, tt.args.osInstance, tt.args.configType); (err != nil) != tt.wantErr { + t.Errorf("updateConfigStatus() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_getOSAndNodeStatus(t *testing.T) { + type args struct { + ctx context.Context + r common.ReadStatusWriter + name types.NamespacedName + hostName string + } + tests := []struct { + name string + args args + wantOS upgradev1.OS + wantNode corev1.Node + }{ + { + name: "get node error", + args: args{ + ctx: context.Background(), + r: &OSReconciler{}, + name: types.NamespacedName{}, + hostName: "test-node", + }, + wantOS: upgradev1.OS{}, + wantNode: corev1.Node{}, + }, + } + patchGet := gomonkey.ApplyMethodSeq(&OSReconciler{}, "Get", []gomonkey.OutputCell{ + {Values: gomonkey.Params{nil}}, + {Values: gomonkey.Params{fmt.Errorf("get node error")}}, + }) + defer patchGet.Reset() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotOS, gotNode := getOSAndNodeStatus(tt.args.ctx, tt.args.r, tt.args.name, tt.args.hostName) + if !reflect.DeepEqual(gotOS, tt.wantOS) { + t.Errorf("getOSAndNodeStatus() gotOS = %v, want %v", gotOS, tt.wantOS) + } + if !reflect.DeepEqual(gotNode, tt.wantNode) { + t.Errorf("getOSAndNodeStatus() gotNode = %v, want %v", gotNode, tt.wantNode) + } + }) + } +} -- Gitee From c7803860105be26a340641c2f505fc6c142deada Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 5 Sep 2023 12:38:18 +0800 Subject: [PATCH 22/27] KubeOS: fix proxy upgrade requeue bug fix the bug that proxy doesnt forget ratelimit after updating config Signed-off-by: Yuhang Wei --- cmd/proxy/controllers/os_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index b8d0f802..0630a98c 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -101,7 +101,7 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re if err = r.Update(ctx, osInstance); err != nil { return values.RequeueNow, err } - return values.RequeueNow, nil + return values.Requeue, nil } if err := r.setConfig(ctx, osInstance, values.SysConfigName); err != nil { return values.RequeueNow, err -- Gitee From ff9bc2aaf1ee8150652cd62e4d357f31db9e7766 Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Tue, 5 Sep 2023 17:01:41 +0800 Subject: [PATCH 23/27] KubeOS: fix label changed after upgrade When upgrading, the label of the new root partition are changed to the original disk information Signed-off-by: Yuhang Wei --- cmd/agent/server/utils.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/agent/server/utils.go b/cmd/agent/server/utils.go index d2d09463..b4a19ffd 100644 --- a/cmd/agent/server/utils.go +++ b/cmd/agent/server/utils.go @@ -75,6 +75,9 @@ func deleteNewline(out string) string { } func install(imagePath string, side string, next string) error { + if err := modifyImageLabel(imagePath, side, next); err != nil { + return err + } if err := runCommand("dd", "if="+imagePath, "of="+side, "bs=8M"); err != nil { return err } @@ -370,3 +373,10 @@ func getOCIImageDigest(containerRuntime string, imageName string) (string, error } return imageDigests, nil } + +func modifyImageLabel(imagePath, side, next string) error { + if err := runCommand("e2label", imagePath, "ROOT-"+next); err != nil { + return err + } + return nil +} -- Gitee From b931854d11abc9324d8fec4bc0aaae484016149a Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Thu, 7 Sep 2023 11:05:05 +0800 Subject: [PATCH 24/27] KubeOS: fix configs not sync bug Fix data did not sync to the disk powering down happend in configuration Signed-off-by: Yuhang Wei --- cmd/agent/server/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index f186ee6e..a3535008 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -305,6 +305,9 @@ func writeConfigToFile(path string, configs []string) error { if err = w.Flush(); err != nil { return err } + if err := f.Sync(); err != nil { + return err + } return nil } -- Gitee From 0cfab4c43363c4b5889a83b560f7c69a7e59430e Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Mon, 11 Sep 2023 11:20:08 +0800 Subject: [PATCH 25/27] KubeOS: fix log printing order Signed-off-by: Yuhang Wei --- cmd/agent/server/config.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/agent/server/config.go b/cmd/agent/server/config.go index a3535008..ee4297ab 100644 --- a/cmd/agent/server/config.go +++ b/cmd/agent/server/config.go @@ -348,7 +348,7 @@ func getGrubCfgPath() string { func handleDeleteKey(config []string, configInfo *agent.KeyInfo) string { key := config[0] if len(config) == onlyKey && configInfo.Value == "" { - logrus.Infoln("delete configuration ", key) + logrus.Infoln("delete configuration", key) return "" } else if len(config) == onlyKey && configInfo.Value != "" { logrus.Warnf("Failed to delete key %s with inconsistent values "+ @@ -398,14 +398,14 @@ func handleUpdateKey(config []string, configInfo *agent.KeyInfo, isFound bool) s func handleAddKey(m map[string]*agent.KeyInfo, isOnlyKeyValid bool) []string { var configs []string for key, keyInfo := range m { - if key == "" || strings.Contains(key, "=") { - logrus.Warnf("Failed to add nil key or key containing =, key: %s", key) - continue - } if keyInfo.Operation == "delete" { logrus.Warnf("Failed to delete inexistent key %s", key) continue } + if key == "" || strings.Contains(key, "=") { + logrus.Warnf("Failed to add nil key or key containing =, key: %s", key) + continue + } if keyInfo.Operation != "" { logrus.Warnf("Unknown operation %s, adding key %s with value %s by default", keyInfo.Operation, key, keyInfo.Value) -- Gitee From ae931086fcb96c068f77bfa7c6c1539b43489fab Mon Sep 17 00:00:00 2001 From: qiujiacai Date: Wed, 20 Sep 2023 16:30:15 +0800 Subject: [PATCH 26/27] KubeOS:optimized func getNextPart() --- cmd/agent/server/utils.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/agent/server/utils.go b/cmd/agent/server/utils.go index b4a19ffd..fdddc7d5 100644 --- a/cmd/agent/server/utils.go +++ b/cmd/agent/server/utils.go @@ -101,12 +101,10 @@ func getNextPart(partA string, partB string) (string, string, error) { mountPoint := strings.TrimSpace(string(out)) side := partA + next := "A" if mountPoint == "/" { side = partB - } - next := "B" - if side != partB { - next = "A" + next = "B" } return side, next, nil } -- Gitee From e2f62abf1669ed7c644b88409f7a448f01f75b74 Mon Sep 17 00:00:00 2001 From: liyuanr Date: Mon, 25 Sep 2023 10:52:29 +0800 Subject: [PATCH 27/27] KubeOS:add nodeselector to OS The nodeselector field is added to the OS to filter the nodes to be upgraded or configured. During nodeselector configuration, only nodes with this label can be upgraded or configured. Signed-off-by: liyuanr --- api/v1alpha1/os_types.go | 2 + cmd/operator/controllers/operation.go | 181 ++++++ cmd/operator/controllers/os_controller.go | 228 +++----- .../controllers/os_controller_test.go | 550 ++++++++++++++---- cmd/proxy/controllers/os_controller.go | 7 + .../config/crd/upgrade.openeuler.org_os.yaml | 2 + .../config/samples/upgrade_v1alpha1_os.yaml | 3 +- docs/quick-start.md | 55 +- pkg/values/values.go | 6 +- 9 files changed, 748 insertions(+), 286 deletions(-) create mode 100644 cmd/operator/controllers/operation.go diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go index f9474b72..d3d636de 100644 --- a/api/v1alpha1/os_types.go +++ b/api/v1alpha1/os_types.go @@ -38,6 +38,8 @@ type OSSpec struct { SysConfigs SysConfigs `json:"sysconfigs"` // +kubebuilder:validation:Optional UpgradeConfigs SysConfigs `json:"upgradeconfigs"` + // +kubebuilder:validation:Optional + NodeSelector string `json:"nodeselector"` } // +kubebuilder:subresource:status diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go new file mode 100644 index 00000000..4b441d18 --- /dev/null +++ b/cmd/operator/controllers/operation.go @@ -0,0 +1,181 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package controllers contains the Reconcile of operator +package controllers + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/selection" + "k8s.io/apimachinery/pkg/types" + upgradev1 "openeuler.org/KubeOS/api/v1alpha1" + "openeuler.org/KubeOS/pkg/common" + "openeuler.org/KubeOS/pkg/values" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type operation interface { + newExistRequirement() (labels.Requirement, error) + newNotExistRequirement() (labels.Requirement, error) + updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) + updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error +} + +type upgradeOps struct{} + +func (u upgradeOps) newExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (u upgradeOps) newNotExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelUpgrading) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) { + var count int + for _, node := range nodes { + if count >= limit { + break + } + osVersionNode := node.Status.NodeInfo.OSImage + if os.Spec.OSVersion != osVersionNode { + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "failed to get osInstance "+node.Name, "skip this node") + return count, err + } + continue + } + if err := u.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + continue + } + count++ + } + } + return count, nil +} + +func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error { + if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { + if err := deepCopySpecConfigs(os, osInstance, values.UpgradeConfigName); err != nil { + return err + } + } + if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { + if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { + return err + } + // exchange "grub.cmdline.current" and "grub.cmdline.next" + for i, config := range osInstance.Spec.SysConfigs.Configs { + if config.Model == "grub.cmdline.current" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" + } + if config.Model == "grub.cmdline.next" { + osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" + } + } + } + osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } + node.Labels[values.LabelUpgrading] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } + return nil +} + +type configOps struct{} + +func (c configOps) newExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.Exists, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelConfiguring) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (c configOps) newNotExistRequirement() (labels.Requirement, error) { + requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.DoesNotExist, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelConfiguring) + return labels.Requirement{}, err + } + return *requirement, nil +} + +func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + nodes []corev1.Node, limit int) (int, error) { + var count int + for _, node := range nodes { + if count >= limit { + break + } + var osInstance upgradev1.OSInstance + if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { + if err = client.IgnoreNotFound(err); err != nil { + log.Error(err, "failed to get osInstance "+node.Name) + return count, err + } + continue + } + if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version { + if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { + log.Error(err, "failed to update node and osinstance ,skip this node ") + continue + } + count++ + } + } + return count, nil +} + +func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, + node *corev1.Node, osInstance *upgradev1.OSInstance) error { + if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { + return err + } + osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() + if err := r.Update(ctx, osInstance); err != nil { + log.Error(err, "unable to update", "osInstance", osInstance.Name) + return err + } + node.Labels[values.LabelConfiguring] = "" + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to label", "node", node.Name) + return err + } + return nil +} diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go index e04d59b1..bd7a70dd 100644 --- a/cmd/operator/controllers/os_controller.go +++ b/cmd/operator/controllers/os_controller.go @@ -17,6 +17,7 @@ import ( "context" "encoding/json" "fmt" + "reflect" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -65,29 +66,24 @@ func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) } ops := os.Spec.OpsType + var opsInsatnce operation switch ops { case "upgrade", "rollback": - limit, err := checkUpgrading(ctx, r, min(os.Spec.MaxUnavailable, nodeNum)) // adjust maxUnavailable if need - if err != nil { - return values.RequeueNow, err - } - if needRequeue, err := assignUpgrade(ctx, r, os, limit, req.Namespace); err != nil { - return values.RequeueNow, err - } else if needRequeue { - return values.Requeue, nil - } + opsInsatnce = upgradeOps{} case "config": - limit, err := checkConfig(ctx, r, min(os.Spec.MaxUnavailable, nodeNum)) - if err != nil { - return values.RequeueNow, err - } - if needRequeue, err := assignConfig(ctx, r, os.Spec.SysConfigs, os.Spec.SysConfigs.Version, limit); err != nil { - return values.RequeueNow, err - } else if needRequeue { - return values.Requeue, nil - } + opsInsatnce = configOps{} default: log.Error(nil, "operation "+ops+" cannot be recognized") + return values.Requeue, nil + } + limit, err := calNodeLimit(ctx, r, opsInsatnce, min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need + if err != nil { + return values.RequeueNow, err + } + if needRequeue, err := assignOperation(ctx, r, os, limit, opsInsatnce); err != nil { + return values.RequeueNow, err + } else if needRequeue { + return values.Requeue, nil } return values.Requeue, nil } @@ -129,30 +125,44 @@ func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLim } } -func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (os upgradev1.OS, - nodeNum int, err error) { - if err = r.Get(ctx, name, &os); err != nil { +func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, + int, error) { + var os upgradev1.OS + if err := r.Get(ctx, name, &os); err != nil { log.Error(err, "unable to fetch OS") - return + return upgradev1.OS{}, 0, err } requirement, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil) if err != nil { log.Error(err, "unable to create requirement "+values.LabelMaster) - return + return upgradev1.OS{}, 0, err } - nodesItems, err := getNodes(ctx, r, 0, *requirement) + var requirements []labels.Requirement + requirements = append(requirements, *requirement) + if os.Spec.NodeSelector != "" { + reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Exists, nil) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return upgradev1.OS{}, 0, err + } + requirements = append(requirements, *requirement, *reqSelector) + } + nodesItems, err := getNodes(ctx, r, 0, requirements...) if err != nil { log.Error(err, "get slave nodes fail") - return + return upgradev1.OS{}, 0, err } - nodeNum = len(nodesItems) - return + nodeNum := len(nodesItems) + return os, nodeNum, nil } -func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, - nameSpace string) (bool, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil) +func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int, + ops operation) (bool, error) { + fmt.Println("start assignOperation") + fmt.Println("ops is ", reflect.TypeOf(ops)) + requirement, err := ops.newNotExistRequirement() + fmt.Println("requirement is ", requirement.String()) if err != nil { log.Error(err, "unable to create requirement "+values.LabelUpgrading) return false, err @@ -162,14 +172,25 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. log.Error(err, "unable to create requirement "+values.LabelMaster) return false, err } + var requirements []labels.Requirement + requirements = append(requirements, requirement, *reqMaster) + if os.Spec.NodeSelector != "" { + reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{os.Spec.NodeSelector}) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return false, err + } + fmt.Println("requirement is ", reqSelector.String()) + requirements = append(requirements, *reqSelector) + } - nodes, err := getNodes(ctx, r, limit+1, *requirement, *reqMaster) // one more to see if all nodes updated + nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated if err != nil { return false, err } - + fmt.Println("nodes has not upgrade/config and has selector ", len(nodes)) // Upgrade OS for selected nodes - count, err := upgradeNodes(ctx, r, &os, nodes, limit) + count, err := ops.updateNodes(ctx, r, &os, nodes, limit) if err != nil { return false, err } @@ -177,90 +198,6 @@ func assignUpgrade(ctx context.Context, r common.ReadStatusWriter, os upgradev1. return count >= limit, nil } -func upgradeNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - nodes []corev1.Node, limit int) (int, error) { - var count int - for _, node := range nodes { - if count >= limit { - break - } - osVersionNode := node.Status.NodeInfo.OSImage - if os.Spec.OSVersion != osVersionNode { - var osInstance upgradev1.OSInstance - if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil { - if err = client.IgnoreNotFound(err); err != nil { - log.Error(err, "failed to get osInstance "+node.Name) - return count, err - } - continue - } - if err := updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil { - continue - } - count++ - } - } - return count, nil -} - -func updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS, - node *corev1.Node, osInstance *upgradev1.OSInstance) error { - if osInstance.Spec.UpgradeConfigs.Version != os.Spec.UpgradeConfigs.Version { - if err := deepCopySpecConfigs(os, osInstance, values.UpgradeConfigName); err != nil { - return err - } - } - if osInstance.Spec.SysConfigs.Version != os.Spec.SysConfigs.Version { - if err := deepCopySpecConfigs(os, osInstance, values.SysConfigName); err != nil { - return err - } - // exchange "grub.cmdline.current" and "grub.cmdline.next" - for i, config := range osInstance.Spec.SysConfigs.Configs { - if config.Model == "grub.cmdline.current" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.next" - } - if config.Model == "grub.cmdline.next" { - osInstance.Spec.SysConfigs.Configs[i].Model = "grub.cmdline.current" - } - } - } - osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String() - if err := r.Update(ctx, osInstance); err != nil { - log.Error(err, "unable to update", "osInstance", osInstance.Name) - return err - } - node.Labels[values.LabelUpgrading] = "" - if err := r.Update(ctx, node); err != nil { - log.Error(err, "unable to label", "node", node.Name) - return err - } - return nil -} - -func assignConfig(ctx context.Context, r common.ReadStatusWriter, sysConfigs upgradev1.SysConfigs, - configVersion string, limit int) (bool, error) { - osInstances, err := getIdleOSInstances(ctx, r, limit+1) // one more to see if all node updated - if err != nil { - return false, err - } - var count = 0 - for _, osInstance := range osInstances { - if count >= limit { - break - } - configVersionNode := osInstance.Spec.SysConfigs.Version - if configVersion != configVersionNode { - count++ - osInstance.Spec.SysConfigs = sysConfigs - osInstance.Spec.NodeStatus = values.NodeStatusConfig.String() - if err = r.Update(ctx, &osInstance); err != nil { - log.Error(err, "unable update osInstance ", "osInstanceName ", osInstance.Name) - } - } - } - return count >= limit, nil -} - func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, reqs ...labels.Requirement) ([]corev1.Node, error) { var nodeList corev1.NodeList @@ -272,48 +209,37 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int, return nodeList.Items, nil } -func getIdleOSInstances(ctx context.Context, r common.ReadStatusWriter, limit int) ([]upgradev1.OSInstance, error) { - var osInstanceList upgradev1.OSInstanceList - opt := []client.ListOption{ - client.MatchingFields{values.OsiStatusName: values.NodeStatusIdle.String()}, - &client.ListOptions{Limit: int64(limit)}, - } - if err := r.List(ctx, &osInstanceList, opt...); err != nil { - log.Error(err, "unable to list nodes with requirements") - return nil, err - } - return osInstanceList.Items, nil -} - -func getConfigOSInstances(ctx context.Context, r common.ReadStatusWriter) ([]upgradev1.OSInstance, error) { - var osInstanceList upgradev1.OSInstanceList - if err := r.List(ctx, &osInstanceList, - client.MatchingFields{values.OsiStatusName: values.NodeStatusConfig.String()}); err != nil { - log.Error(err, "unable to list nodes with requirements") - return nil, err - } - return osInstanceList.Items, nil -} - -func checkUpgrading(ctx context.Context, r common.ReadStatusWriter, maxUnavailable int) (int, error) { - requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil) +func calNodeLimit(ctx context.Context, r common.ReadStatusWriter, + ops operation, maxUnavailable int, nodeSelector string) (int, error) { + fmt.Println("start calNodeLimit") + fmt.Println("ops is ", reflect.TypeOf(ops)) + requirement, err := ops.newExistRequirement() if err != nil { log.Error(err, "unable to create requirement "+values.LabelUpgrading) return 0, err } - nodes, err := getNodes(ctx, r, 0, *requirement) - if err != nil { - return 0, err + fmt.Println("requirement is ", requirement.String()) + var requirements []labels.Requirement + requirements = append(requirements, requirement) + if nodeSelector != "" { + reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{nodeSelector}) + if err != nil { + log.Error(err, "unable to create requirement "+values.LabelNodeSelector) + return 0, err + } + fmt.Println("requirement is ", reqSelector.String()) + requirements = append(requirements, *reqSelector) } - return maxUnavailable - len(nodes), nil -} - -func checkConfig(ctx context.Context, r common.ReadStatusWriter, maxUnavailable int) (int, error) { - osInstances, err := getConfigOSInstances(ctx, r) + nodes, err := getNodes(ctx, r, 0, requirements...) if err != nil { return 0, err + } - return maxUnavailable - len(osInstances), nil + fmt.Println("nodes has upgrade and selector ", len(nodes)) + for _, n := range nodes { + fmt.Println(" nodes name is ", n.Name) + } + return maxUnavailable - len(nodes), nil } func min(a, b int) int { diff --git a/cmd/operator/controllers/os_controller_test.go b/cmd/operator/controllers/os_controller_test.go index 6cc2760e..98afb26c 100644 --- a/cmd/operator/controllers/os_controller_test.go +++ b/cmd/operator/controllers/os_controller_test.go @@ -456,6 +456,14 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(configedOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) Expect(configedOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + existingNode1 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode1) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode1.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(true)) configedOSIns2 := &upgradev1.OSInstance{} Eventually(func() bool { @@ -464,6 +472,14 @@ var _ = Describe("OsController", func() { }, timeout, interval).Should(BeTrue()) Expect(configedOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) Expect(configedOSIns2.Spec.SysConfigs.Version).Should(Equal("v2")) + existingNode2 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok = existingNode2.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(true)) }) }) @@ -750,6 +766,419 @@ var _ = Describe("OsController", func() { Expect(createdOS.Spec.SysConfigs.Configs[1]).Should(Equal(upgradev1.SysConfig{Model: "grub.cmdline.next", Contents: []upgradev1.Content{{Key: "b", Value: "2"}}})) }) }) + + Context("When we want to upgrade node with nodes having NodeSelector label", func() { + It("Should only update node with NodeSelector label", func() { + ctx := context.Background() + // create Node1 + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + "upgrade.openeuler.org/node-selector": "openeuler", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance1 + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{ + Version: "v1", + Configs: []upgradev1.SysConfig{}, + }, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "upgrade", + MaxUnavailable: 3, + OSVersion: "KubeOS v2", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + NodeSelector: "openeuler", + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }, + }, + }, + UpgradeConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + {Model: "kernel.sysctl.persist", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }}, + }, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.OSVersion).Should(Equal("KubeOS v2")) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + existingNode1 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode1) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode1.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(true)) + + upgradeOSIns1 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, upgradeOSIns1) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusUpgrade.String())) + Expect(upgradeOSIns1.Spec.UpgradeConfigs.Version).Should(Equal("v2")) + Expect(upgradeOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + + existingNode2 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok = existingNode2.Labels[values.LabelUpgrading] + Expect(ok).Should(Equal(false)) + + upgradeOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, upgradeOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + Expect(upgradeOSIns2.Spec.UpgradeConfigs.Version).Should(Equal("v1")) + Expect(upgradeOSIns2.Spec.SysConfigs.Version).Should(Equal("v1")) + }) + }) + + Context("When we want to config node with nodes having NodeSelector label", func() { + It("Should only config node with NodeSelector label", func() { + ctx := context.Background() + // create Node1 + node1Name = "test-node-" + uuid.New().String() + node1 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + "upgrade.openeuler.org/node-selector": "openeuler", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err := k8sClient.Create(ctx, node1) + Expect(err).ToNot(HaveOccurred()) + existingNode := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance1 + OSIns := &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node1Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node1Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + osInsCRLookupKey1 := types.NamespacedName{Name: node1Name, Namespace: testNamespace} + createdOSIns := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node1Name)) + + // create Node2 + node2Name := "test-node-" + uuid.New().String() + node2 := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + "beta.kubernetes.io/os": "linux", + }, + }, + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Node", + }, + Status: v1.NodeStatus{ + NodeInfo: v1.NodeSystemInfo{ + OSImage: "KubeOS v1", + }, + }, + } + err = k8sClient.Create(ctx, node2) + Expect(err).ToNot(HaveOccurred()) + existingNode = &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode) + return err == nil + }, timeout, interval).Should(BeTrue()) + + // create OSInstance2 + OSIns = &upgradev1.OSInstance{ + TypeMeta: metav1.TypeMeta{ + Kind: "OSInstance", + APIVersion: "upgrade.openeuler.org/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: node2Name, + Namespace: testNamespace, + Labels: map[string]string{ + values.LabelOSinstance: node2Name, + }, + }, + Spec: upgradev1.OSInstanceSpec{ + SysConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + UpgradeConfigs: upgradev1.SysConfigs{Configs: []upgradev1.SysConfig{}, Version: "v1"}, + NodeStatus: values.NodeStatusIdle.String(), + }, + } + Expect(k8sClient.Create(ctx, OSIns)).Should(Succeed()) + + // Check that the corresponding OSIns CR has been created + osInsCRLookupKey2 := types.NamespacedName{Name: node2Name, Namespace: testNamespace} + createdOSIns = &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, createdOSIns) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOSIns.ObjectMeta.Name).Should(Equal(node2Name)) + + OS := &upgradev1.OS{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "upgrade.openeuler.org/v1alpha1", + Kind: "OS", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: OSName, + Namespace: testNamespace, + }, + Spec: upgradev1.OSSpec{ + OpsType: "config", + MaxUnavailable: 3, + OSVersion: "KubeOS v1", + FlagSafe: true, + MTLS: false, + EvictPodForce: true, + NodeSelector: "openeuler", + SysConfigs: upgradev1.SysConfigs{ + Version: "v2", + Configs: []upgradev1.SysConfig{ + { + Model: "kernel.sysctl", + Contents: []upgradev1.Content{ + {Key: "key1", Value: "a"}, + {Key: "key2", Value: "b"}, + }, + }, + }, + }, + }, + } + Expect(k8sClient.Create(ctx, OS)).Should(Succeed()) + + osCRLookupKey := types.NamespacedName{Name: OSName, Namespace: testNamespace} + createdOS := &upgradev1.OS{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osCRLookupKey, createdOS) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(createdOS.Spec.SysConfigs.Version).Should(Equal("v2")) + + time.Sleep(1 * time.Second) // sleep a while to make sure Reconcile finished + existingNode1 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node1Name, Namespace: testNamespace}, existingNode1) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok := existingNode1.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(true)) + + upgradeOSIns1 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey1, upgradeOSIns1) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns1.Spec.NodeStatus).Should(Equal(values.NodeStatusConfig.String())) + Expect(upgradeOSIns1.Spec.SysConfigs.Version).Should(Equal("v2")) + + existingNode2 := &v1.Node{} + Eventually(func() bool { + err := k8sClient.Get(context.Background(), + types.NamespacedName{Name: node2Name, Namespace: testNamespace}, existingNode2) + return err == nil + }, timeout, interval).Should(BeTrue()) + _, ok = existingNode2.Labels[values.LabelConfiguring] + Expect(ok).Should(Equal(false)) + + upgradeOSIns2 := &upgradev1.OSInstance{} + Eventually(func() bool { + err := k8sClient.Get(ctx, osInsCRLookupKey2, upgradeOSIns2) + return err == nil + }, timeout, interval).Should(BeTrue()) + Expect(upgradeOSIns2.Spec.NodeStatus).Should(Equal(values.NodeStatusIdle.String())) + Expect(upgradeOSIns2.Spec.SysConfigs.Version).Should(Equal("v1")) + }) + }) }) func Test_deepCopySpecConfigs(t *testing.T) { @@ -781,127 +1210,6 @@ func Test_deepCopySpecConfigs(t *testing.T) { } } -func Test_getConfigOSInstances(t *testing.T) { - type args struct { - ctx context.Context - r common.ReadStatusWriter - } - tests := []struct { - name string - args args - want []upgradev1.OSInstance - wantErr bool - }{ - { - name: "list error", - args: args{ - ctx: context.Background(), - r: &OSReconciler{}, - }, - want: nil, - wantErr: true, - }, - } - patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ - {Values: gomonkey.Params{fmt.Errorf("list error")}}, - }) - defer patchList.Reset() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := getConfigOSInstances(tt.args.ctx, tt.args.r) - if (err != nil) != tt.wantErr { - t.Errorf("getConfigOSInstances() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getConfigOSInstances() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_checkUpgrading(t *testing.T) { - type args struct { - ctx context.Context - r common.ReadStatusWriter - maxUnavailable int - } - tests := []struct { - name string - args args - want int - wantErr bool - }{ - { - name: "label error", - args: args{ - ctx: context.Background(), - r: &OSReconciler{}, - }, - want: 0, - wantErr: true, - }, - } - patchNewRequirement := gomonkey.ApplyFuncSeq(labels.NewRequirement, []gomonkey.OutputCell{ - {Values: gomonkey.Params{nil, fmt.Errorf("label error")}}, - {Values: gomonkey.Params{nil, nil}}, - }) - defer patchNewRequirement.Reset() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := checkUpgrading(tt.args.ctx, tt.args.r, tt.args.maxUnavailable) - if (err != nil) != tt.wantErr { - t.Errorf("checkUpgrading() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("checkUpgrading() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_getIdleOSInstances(t *testing.T) { - type args struct { - ctx context.Context - r common.ReadStatusWriter - limit int - } - tests := []struct { - name string - args args - want []upgradev1.OSInstance - wantErr bool - }{ - { - name: "list error", - args: args{ - ctx: context.Background(), - r: &OSReconciler{}, - limit: 1, - }, - want: nil, - wantErr: true, - }, - } - patchList := gomonkey.ApplyMethodSeq(&OSReconciler{}, "List", []gomonkey.OutputCell{ - {Values: gomonkey.Params{fmt.Errorf("list error")}}, - }) - defer patchList.Reset() - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := getIdleOSInstances(tt.args.ctx, tt.args.r, tt.args.limit) - if (err != nil) != tt.wantErr { - t.Errorf("getIdleOSInstances() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("getIdleOSInstances() = %v, want %v", got, tt.want) - } - }) - } -} - func Test_getNodes(t *testing.T) { type args struct { ctx context.Context diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index 0630a98c..b543befc 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -244,6 +244,13 @@ func (r *OSReconciler) refreshNode(ctx context.Context, node *corev1.Node, osIns return err } } + if _, ok := node.Labels[values.LabelConfiguring]; ok { + delete(node.Labels, values.LabelConfiguring) + if err := r.Update(ctx, node); err != nil { + log.Error(err, "unable to delete label", "node", node.Name) + return err + } + } if node.Spec.Unschedulable { // update done, uncordon the node drainer := &drain.Helper{ Ctx: ctx, diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index 3bb1333d..acc70fc0 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -54,6 +54,8 @@ spec: type: integer mtls: type: boolean + nodeselector: + type: string opstype: type: string osversion: diff --git a/docs/example/config/samples/upgrade_v1alpha1_os.yaml b/docs/example/config/samples/upgrade_v1alpha1_os.yaml index b33a30b2..ca3da1b2 100644 --- a/docs/example/config/samples/upgrade_v1alpha1_os.yaml +++ b/docs/example/config/samples/upgrade_v1alpha1_os.yaml @@ -13,6 +13,7 @@ spec: checksum: image digests flagSafe: false mtls: false + nodeselector: edit.nodes.label sysconfigs: version: edit.sysconfig.version configs: @@ -35,4 +36,4 @@ spec: contents: - key: kernel param key4 value: kernel param value4 - operation: delete \ No newline at end of file + operation: delete diff --git a/docs/quick-start.md b/docs/quick-start.md index 0d4dc4bf..748e0819 100644 --- a/docs/quick-start.md +++ b/docs/quick-start.md @@ -176,6 +176,7 @@ | clientcert | string | https双向认证时使用的客户端证书文件 | 仅在使用https双向认证时有效|mtls为true时必选 | | clientkey | string | https双向认证时使用的客户端公钥 | 仅在使用https双向认证时有效|mtls为true时必选 | | evictpodforce | bool | 用于表示升级/回退时是否强制驱逐pod | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 | + | nodeselector | string | 需要进行升级/配置/回滚操作的节点label | 用于只对具有某些特定label的节点而不是集群所有worker节点进行运维的场景，需要进行运维操作的节点需要包含key为upgrade.openeuler.org/node-selector的label，nodeselector为该label的value值，此参数不配置时，或者配置为""时默认对所有节点进行操作| 可选 | | sysconfigs | / | 需要进行配置的参数值 | 在配置或者升级或者回退机器时有效，在升级或者回退操作之后即机器重启之后起效，详细字段说明请见```配置（Settings）指导```| 可选 | | upgradeconfigs | / | 需要升级前进行的配置的参数值 | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 | @@ -247,7 +248,7 @@ mtls: true ``` - * 升级并且进行配置的示例如下， + * 升级并且进行配置的示例如下 * 以节点容器引擎为containerd为例，升级方式对配置无影响，upgradeconfigs在升级前起效，sysconfigs在升级后起效，配置参数说明请见```配置(Settings)指导``` * 升级并且配置时opstype字段需为upgrade * upgradeconfig为升级之前执行的配置，sysconfigs为升级机器重启后执行的配置，用户可按需进行配置 @@ -292,7 +293,37 @@ - key: kernel param key4 value: kernel param value4 ``` + * 只升级部分节点示例如下 + * 以节点容器引擎为containerd为例，升级方式对节点筛选无影响 + * 需要进行升级的节点需包含key为upgrade.openeuler.org/node-selector的label，nodeselector的值为该label的value，即假定nodeselector值为kubeos，则只对包含upgrade.openeuler.org/node-selector=kubeos的label的worker节点进行升级 + * nodeselector对配置和回滚同样有效 + * 节点添加label和label修改命令示例如下： + ``` shell + # 为节点kubeos-node1增加label + kubectl label nodes kubeos-node1 upgrade.openeuler.org/node-selector=kubeos-v1 + # 修改节点kubeos-node1的label + kubectl label --overwrite nodes kubeos-node2 upgrade.openeuler.org/node-selector=kubeos-v2 + ``` + * yaml示例如下： + ```yaml + apiVersion: upgrade.openeuler.org/v1alpha1 + kind: OS + metadata: + name: os-sample + spec: + imagetype: containerd + opstype: upgrade + osversion: edit.os.version + maxunavailable: edit.node.upgrade.number + containerimage: container image like repository/name:tag + evictpodforce: true/false + imageurl: "" + checksum: container image digests + flagSafe: false + mtls: true + nodeselector: edit.node.label.key + ``` * 查看未升级的节点的 OS 版本 ```shell @@ -366,20 +397,20 @@ * 查看配置之前的节点的配置的版本和节点状态（NODESTATUS状态为idle） ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' ``` * 执行命令，在集群中部署cr实例后，节点会根据配置的参数信息进行配置，再次查看节点状态(NODESTATUS变成config) ```shell kubectl apply -f upgrade_v1alpha1_os.yaml - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' ``` * 再次查看节点的配置的版本确认节点是否配置完成(NODESTATUS恢复为idle) ```shell - kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version' + kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version' ``` * 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。 @@ -605,7 +636,7 @@ hostshell #### kernel Settings -* kenerl.sysctl: 设置内核参数，key/value 表示内核参数的 key/value, 示例如下: +* kenerl.sysctl: 临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）, 示例如下: ```yaml configs: @@ -618,8 +649,7 @@ hostshell operation: delete ``` -* kernel.sysctl.persist: 设置持久化内核参数，key/value 表示内核参数的 key/value, configpath为配置修改/新建的文件路径，如不指定configpath默认修改/etc/sysctl.conf - +* kernel.sysctl.persist: 设置持久化内核参数，key/value表示内核参数的key/value，key与value均不能为空且key不能包含“=”， configpath为配置文件路径，支持新建（需保证父目录存在），如不指定configpath默认修改/etc/sysctl.conf，示例如下： ```yaml configs: - model: kernel.systcl.persist @@ -637,13 +667,14 @@ hostshell * grub.cmdline: 设置grub.cfg文件中的内核引导参数，该行参数在grub.cfg文件中类似如下示例： ```shell - linux /boot/vmlinuz root=UUID=5b1aaf5d-5b25-4e4b-a0d3-3d4c8d2e6a6e ro consoleblank=600 console=tty0 console=ttyS0,115200n8 selinux=1 panic=3 + linux /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3 ``` - key/value 表示如上示例中内核引导参数的 key=value。 - **注意：** 当该参数有多个等号，如root=UUID=some-uuid时，配置时的key为第一个等号前的所有字符，value为第一个等号后的所有字符。 - 配置方法示例如下： - +* KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置： + * grub.cmdline.current：对当前分区的启动项参数进行配置。 + * grub.cmdline.next：对下一分区的启动项参数进行配置。 +* 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。 +* grub.cmdline.current/next支持“key=value”（value不能为空），也支持单key。若value中有“=”，例如“root=UUID=some-uuid”，key应设置为第一个“=”前的所有字符，value为第一个“=”后的所有字符。 配置方法示例如下： ```yaml configs: - model: grub.cmdline diff --git a/pkg/values/values.go b/pkg/values/values.go index f488ae52..e2c03540 100644 --- a/pkg/values/values.go +++ b/pkg/values/values.go @@ -26,7 +26,11 @@ const ( LabelMaster = "node-role.kubernetes.io/control-plane" // LabelOSinstance is used to select the osinstance with the nodeName by label LabelOSinstance = "upgrade.openeuler.org/osinstance-node" - defaultPeriod = 15 * time.Second + // LabelNodeSelector is used to filter the nodes that need to be upgraded or configured. + LabelNodeSelector = "upgrade.openeuler.org/node-selector" + // LabelConfiguring is the key of the configuring label for nodes + LabelConfiguring = "upgrade.openeuler.org/configuring" + defaultPeriod = 15 * time.Second // OsiStatusName is param name of nodeStatus in osInstance OsiStatusName = "nodestatus" // UpgradeConfigName is param name of UpgradeConfig -- Gitee