From c212b7e4165e88a3f162ed5a0edee3c611614aa0 Mon Sep 17 00:00:00 2001 From: lauk Date: Wed, 12 Jul 2023 15:37:50 +0800 Subject: [PATCH] Optimize the drain strategy of the controller --- housekeeper/daemon/server/server.go | 16 +---- .../controllers/update_controller.go | 60 +++++++++++-------- housekeeper/pkg/common/common.go | 12 ++++ housekeeper/pkg/connection/connection.go | 14 ++--- housekeeper/pkg/connection/proto/daemon.pb.go | 53 +++++++--------- housekeeper/pkg/connection/proto/daemon.proto | 1 - housekeeper/pkg/constants/constants.go | 3 - 7 files changed, 78 insertions(+), 81 deletions(-) diff --git a/housekeeper/daemon/server/server.go b/housekeeper/daemon/server/server.go index de06f32..a85f234 100644 --- a/housekeeper/daemon/server/server.go +++ b/housekeeper/daemon/server/server.go @@ -25,6 +25,7 @@ import ( "sync" "github.com/sirupsen/logrus" + "housekeeper.io/pkg/common" pb "housekeeper.io/pkg/connection/proto" utilVersion "k8s.io/apimachinery/pkg/util/version" ) @@ -54,7 +55,7 @@ func (s *Server) Upgrade(_ context.Context, req *pb.UpgradeRequest) (*pb.Upgrade return &pb.UpgradeResponse{}, err } } - if isFileExist(markFile) { + if common.IsFileExist(markFile) { return &pb.UpgradeResponse{}, nil } // upgrade kubernetes @@ -181,7 +182,7 @@ func upgradeNodes() error { } func isControlPlaneNode() (bool, error) { - if !isFileExist("/etc/kubernetes/admin.conf") { + if !common.IsFileExist("/etc/kubernetes/admin.conf") { return false, nil } ipArgs := []string{"-c", "ifconfig | grep 'inet' | grep 'broadcast'| awk '{print $2}'"} @@ -209,17 +210,6 @@ func markNode(file string) error { return nil } -func isFileExist(path string) bool { - fileInfo, err := os.Stat(path) - if err != nil { - return false - } - if fileInfo.IsDir() { - return false - } - return true -} - func runCmd(name string, args ...string) ([]byte, error) { cmd := exec.Command(name, args...) output, err := cmd.Output() diff --git a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go index 67feb11..6546451 100644 --- a/housekeeper/operator/housekeeper-controller/controllers/update_controller.go +++ b/housekeeper/operator/housekeeper-controller/controllers/update_controller.go @@ -32,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "k8s.io/kubectl/pkg/drain" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -78,7 +79,13 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr _ = log.FromContext(ctx) ctx = context.Background() upInstance, nodeInstance := reqInstance(ctx, r, req.NamespacedName, r.HostName) - upgradeCluster := checkUpgrade(&nodeInstance, upInstance.Spec.OSVersion, upInstance.Spec.KubeVersion) + var ( + osVersionSpec = upInstance.Spec.OSVersion + kubeVersionSpec = upInstance.Spec.KubeVersion + // osVersion reported by the node from /etc/os-release + osVersion = nodeInstance.Status.NodeInfo.OSImage + ) + upgradeCluster := checkUpgrade(osVersion, osVersionSpec, kubeVersionSpec) if upgradeCluster { if err := r.upgradeNodes(ctx, &upInstance, &nodeInstance); err != nil { return common.RequeueNow, err @@ -91,26 +98,24 @@ func (r *UpdateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr func (r *UpdateReconciler) upgradeNodes(ctx context.Context, upInstance *housekeeperiov1alpha1.Update, node *corev1.Node) error { - controlPlane := false - if _, ok := node.Labels[constants.LabelMaster]; ok { - controlPlane = true - } if _, ok := node.Labels[constants.LabelUpgrading]; ok { drainer := &drain.Helper{ - Ctx: ctx, - Client: r.KubeClientSet, - GracePeriodSeconds: -1, - Out: os.Stdout, - ErrOut: os.Stderr, + Ctx: ctx, + Client: r.KubeClientSet, + Force: true, + IgnoreAllDaemonSets: true, + DeleteEmptyDirData: true, + GracePeriodSeconds: -1, + Out: os.Stdout, + ErrOut: os.Stderr, } if err := drainNode(drainer, node); err != nil { return err } pushInfo := &connection.PushInfo{ - KubeVersion: upInstance.Spec.KubeVersion, - OSImageURL: upInstance.Spec.OSImageURL, - OSVersion: upInstance.Spec.OSVersion, - ControlPlane: controlPlane, + KubeVersion: upInstance.Spec.KubeVersion, + OSImageURL: upInstance.Spec.OSImageURL, + OSVersion: upInstance.Spec.OSVersion, } if err := r.Connection.UpgradeKubeSpec(pushInfo); err != nil { return err @@ -124,13 +129,17 @@ func (r *UpdateReconciler) refreshNodes(ctx context.Context, upInstance *houseke deleteLabel(ctx, r, node) if node.Spec.Unschedulable { drainer := &drain.Helper{ - Ctx: ctx, - Client: r.KubeClientSet, - GracePeriodSeconds: -1, - Out: os.Stdout, - ErrOut: os.Stderr, + Ctx: ctx, + Client: r.KubeClientSet, + Force: true, + IgnoreAllDaemonSets: true, + DeleteEmptyDirData: true, + GracePeriodSeconds: -1, + Out: os.Stdout, + ErrOut: os.Stderr, } - if err := drain.cordonOrUncordonNode(false, drainer, node); err != nil { + if err := cordonOrUncordonNode(false, drainer, node); err != nil { + logrus.Errorf("failed to uncordon node %s: %v", node.Name, err) return err } logrus.Infof("uncordon successfully %s node", node.Name) @@ -149,6 +158,7 @@ func deleteLabel(ctx context.Context, r common.ReadWriterClient, node *corev1.No return nil } +// Sets schedulable or not func cordonOrUncordonNode(desired bool, drainer *drain.Helper, node *corev1.Node) error { carry := "cordon" if !desired { @@ -192,15 +202,17 @@ func reqInstance(ctx context.Context, r common.ReadWriterClient, name types.Name return } -func checkUpgrade(node *corev1.Node, osVersionSpec string, kubeVersionSpec string) bool { +// Check if the version is upgraded +func checkUpgrade(osVersion string, osVersionSpec string, kubeVersionSpec string) bool { if len(kubeVersionSpec) > 0 { - labelKubeVersion := fmt.Sprintf("%s%s", constants.LabelKubeVersionPrefix, kubeVersionSpec) - if _, ok := node.Labels[labelKubeVersion]; ok { + markFile := fmt.Sprintf("%s%s%s", "/var/housekeeper/", kubeVersionSpec, ".stamp") + if common.IsFileExist(markFile) { return false } } else { - return node.Status.NodeInfo.OSImage != osVersionSpec + return osVersion != osVersionSpec } + return true } diff --git a/housekeeper/pkg/common/common.go b/housekeeper/pkg/common/common.go index ec07e07..bac1b78 100644 --- a/housekeeper/pkg/common/common.go +++ b/housekeeper/pkg/common/common.go @@ -16,6 +16,7 @@ limitations under the License. package common import ( + "os" "time" ctrl "sigs.k8s.io/controller-runtime" @@ -36,3 +37,14 @@ var ( RequeueNow = ctrl.Result{Requeue: true} RequeueAfter = ctrl.Result{Requeue: true, RequeueAfter: time.Second * 20} ) + +func IsFileExist(path string) bool { + fileInfo, err := os.Stat(path) + if err != nil { + return false + } + if fileInfo.IsDir() { + return false + } + return true +} diff --git a/housekeeper/pkg/connection/connection.go b/housekeeper/pkg/connection/connection.go index 3e5c1c6..546e977 100644 --- a/housekeeper/pkg/connection/connection.go +++ b/housekeeper/pkg/connection/connection.go @@ -32,10 +32,9 @@ type Client struct { } type PushInfo struct { - OSImageURL string - OSVersion string - KubeVersion string - ControlPlane bool + OSImageURL string + OSVersion string + KubeVersion string } // Create a grpc channel @@ -58,10 +57,9 @@ func New(socketAddr string) (*Client, error) { func (c *Client) UpgradeKubeSpec(pushInfo *PushInfo) error { _, err := c.client.Upgrade(context.Background(), &pb.UpgradeRequest{ - KubeVersion: pushInfo.KubeVersion, - OsImageUrl: pushInfo.OSImageURL, - OsVersion: pushInfo.OSVersion, - ControlPlane: pushInfo.ControlPlane, + KubeVersion: pushInfo.KubeVersion, + OsImageUrl: pushInfo.OSImageURL, + OsVersion: pushInfo.OSVersion, }) return err } diff --git a/housekeeper/pkg/connection/proto/daemon.pb.go b/housekeeper/pkg/connection/proto/daemon.pb.go index 9ef7338..0446d5b 100644 --- a/housekeeper/pkg/connection/proto/daemon.pb.go +++ b/housekeeper/pkg/connection/proto/daemon.pb.go @@ -44,10 +44,9 @@ type UpgradeRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - KubeVersion string `protobuf:"bytes,1,opt,name=kube_version,json=kubeVersion,proto3" json:"kube_version,omitempty"` - OsImageUrl string `protobuf:"bytes,2,opt,name=os_image_url,json=osImageUrl,proto3" json:"os_image_url,omitempty"` - OsVersion string `protobuf:"bytes,3,opt,name=os_version,json=osVersion,proto3" json:"os_version,omitempty"` - ControlPlane bool `protobuf:"varint,4,opt,name=control_plane,json=controlPlane,proto3" json:"control_plane,omitempty"` + KubeVersion string `protobuf:"bytes,1,opt,name=kube_version,json=kubeVersion,proto3" json:"kube_version,omitempty"` + OsImageUrl string `protobuf:"bytes,2,opt,name=os_image_url,json=osImageUrl,proto3" json:"os_image_url,omitempty"` + OsVersion string `protobuf:"bytes,3,opt,name=os_version,json=osVersion,proto3" json:"os_version,omitempty"` } func (x *UpgradeRequest) Reset() { @@ -103,13 +102,6 @@ func (x *UpgradeRequest) GetOsVersion() string { return "" } -func (x *UpgradeRequest) GetControlPlane() bool { - if x != nil { - return x.ControlPlane - } - return false -} - type UpgradeResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -161,27 +153,24 @@ var File_daemon_proto protoreflect.FileDescriptor var file_daemon_proto_rawDesc = []byte{ 0x0a, 0x0c, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x06, - 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x22, 0x99, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, - 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6b, 0x75, 0x62, - 0x65, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0b, 0x6b, 0x75, 0x62, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0c, - 0x6f, 0x73, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x73, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x1d, - 0x0a, 0x0a, 0x6f, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x6f, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x23, 0x0a, - 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x5f, 0x70, 0x6c, 0x61, 0x6e, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x50, 0x6c, 0x61, - 0x6e, 0x65, 0x22, 0x23, 0x0a, 0x0f, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x03, 0x65, 0x72, 0x72, 0x32, 0x4e, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, - 0x64, 0x65, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x3c, 0x0a, 0x07, 0x55, 0x70, 0x67, - 0x72, 0x61, 0x64, 0x65, 0x12, 0x16, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, - 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, - 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x25, 0x5a, 0x23, 0x68, 0x6f, 0x75, 0x73, 0x65, - 0x6b, 0x65, 0x65, 0x70, 0x65, 0x72, 0x2e, 0x69, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x63, 0x6f, - 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x22, 0x74, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, + 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x6b, 0x75, 0x62, 0x65, + 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x6b, 0x75, 0x62, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x20, 0x0a, 0x0c, 0x6f, + 0x73, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0a, 0x6f, 0x73, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x1d, 0x0a, + 0x0a, 0x6f, 0x73, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x09, 0x6f, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x23, 0x0a, 0x0f, + 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x72, + 0x72, 0x32, 0x4e, 0x0a, 0x0e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x43, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x12, 0x3c, 0x0a, 0x07, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x12, 0x16, + 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x64, 0x61, 0x65, 0x6d, 0x6f, 0x6e, 0x2e, + 0x55, 0x70, 0x67, 0x72, 0x61, 0x64, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x42, 0x25, 0x5a, 0x23, 0x68, 0x6f, 0x75, 0x73, 0x65, 0x6b, 0x65, 0x65, 0x70, 0x65, 0x72, + 0x2e, 0x69, 0x6f, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/housekeeper/pkg/connection/proto/daemon.proto b/housekeeper/pkg/connection/proto/daemon.proto index 649f714..f4f5c44 100644 --- a/housekeeper/pkg/connection/proto/daemon.proto +++ b/housekeeper/pkg/connection/proto/daemon.proto @@ -28,7 +28,6 @@ message UpgradeRequest { string kube_version = 1; string os_image_url = 2; string os_version = 3; - bool control_plane = 4; } message UpgradeResponse { diff --git a/housekeeper/pkg/constants/constants.go b/housekeeper/pkg/constants/constants.go index ea930a4..4528bda 100644 --- a/housekeeper/pkg/constants/constants.go +++ b/housekeeper/pkg/constants/constants.go @@ -18,8 +18,6 @@ package constants const ( // LabelUpgrading is the key of the upgrading label for nodes LabelUpgrading = "upgrade.housekeeper.io/upgrading" - // LabelKubeVersionPrefix defines the label associated with kubernetes version - LabelKubeVersionPrefix = "upgrade.kubernetes.version.io/" // LabelMaster defines the label associated with master node. LabelMaster = "node-role.kubernetes.io/master" ) @@ -29,4 +27,3 @@ const ( SockDir = "/run/housekeeper-daemon" SockName = "housekeeper-daemon.sock" ) - -- Gitee