From 675f8fc61bce91f5d061473cc740ff36f870d17d Mon Sep 17 00:00:00 2001 From: liyuanr Date: Thu, 30 Mar 2023 20:38:02 +0800 Subject: [PATCH] KubeOS:support containerd, modify the method of docker using and add the eviction config. Support using containerd as container runtime to upgrade with container images To improve compatibility with Docker of different versions, the original method of invoking Docker APIs is changed to using CLI The podevictforce field is added to the crd of the OS to indicate whether to forcibly evict pods on nodes. Signed-off-by: liyuanr --- api/v1alpha1/os_types.go | 3 +- cmd/agent/api/agent.pb.go | 84 ++++----- cmd/agent/api/agent.proto | 2 +- cmd/agent/server/containerd_image.go | 106 +++++++++++ cmd/agent/server/disk_image.go | 21 +++ cmd/agent/server/docker_image.go | 168 +++-------------- cmd/agent/server/server.go | 22 +-- cmd/agent/server/utils.go | 175 ++++++++++++++++++ cmd/proxy/controllers/os_controller.go | 34 ++-- .../config/crd/upgrade.openeuler.org_os.yaml | 7 +- pkg/agentclient/connection.go | 34 ++-- 11 files changed, 424 insertions(+), 232 deletions(-) create mode 100644 cmd/agent/server/containerd_image.go diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go index 862d4080..2b2554e2 100644 --- a/api/v1alpha1/os_types.go +++ b/api/v1alpha1/os_types.go @@ -25,11 +25,12 @@ type OSSpec struct { FlagSafe bool `json:"flagSafe"` MTLS bool `json:"mtls"` ImageType string `json:"imagetype"` - DockerImage string `json:"dockerimage"` + ContainerImage string `json:"containerimage"` OpsType string `json:"opstype"` CaCert string `json:"cacert"` ClientCert string `json:"clientcert"` ClientKey string `json:"clientkey"` + EvictPodForce bool `json:"evictpodforce"` } // +kubebuilder:subresource:status diff --git a/cmd/agent/api/agent.pb.go b/cmd/agent/api/agent.pb.go index 9aee75b3..336224d9 100644 --- a/cmd/agent/api/agent.pb.go +++ b/cmd/agent/api/agent.pb.go @@ -11,8 +11,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.12.3 +// protoc-gen-go v1.28.1 +// protoc v3.14.0 // source: api/agent.proto package agent @@ -40,14 +40,14 @@ type UpdateRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` - ImageUrl string `protobuf:"bytes,2,opt,name=image_url,json=imageUrl,proto3" json:"image_url,omitempty"` - CheckSum string `protobuf:"bytes,3,opt,name=check_sum,json=checkSum,proto3" json:"check_sum,omitempty"` - FlagSafe bool `protobuf:"varint,4,opt,name=flagSafe,proto3" json:"flagSafe,omitempty"` - MTLS bool `protobuf:"varint,5,opt,name=mTLS,proto3" json:"mTLS,omitempty"` - ImageType string `protobuf:"bytes,6,opt,name=image_type,json=imageType,proto3" json:"image_type,omitempty"` - DockerImage string `protobuf:"bytes,7,opt,name=docker_image,json=dockerImage,proto3" json:"docker_image,omitempty"` - Certs *CertsInfo `protobuf:"bytes,8,opt,name=certs,proto3" json:"certs,omitempty"` + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + ImageUrl string `protobuf:"bytes,2,opt,name=image_url,json=imageUrl,proto3" json:"image_url,omitempty"` + CheckSum string `protobuf:"bytes,3,opt,name=check_sum,json=checkSum,proto3" json:"check_sum,omitempty"` + FlagSafe bool `protobuf:"varint,4,opt,name=flagSafe,proto3" json:"flagSafe,omitempty"` + MTLS bool `protobuf:"varint,5,opt,name=mTLS,proto3" json:"mTLS,omitempty"` + ImageType string `protobuf:"bytes,6,opt,name=image_type,json=imageType,proto3" json:"image_type,omitempty"` + ContainerImage string `protobuf:"bytes,7,opt,name=container_image,json=containerImage,proto3" json:"container_image,omitempty"` + Certs *CertsInfo `protobuf:"bytes,8,opt,name=certs,proto3" json:"certs,omitempty"` } func (x *UpdateRequest) Reset() { @@ -124,9 +124,9 @@ func (x *UpdateRequest) GetImageType() string { return "" } -func (x *UpdateRequest) GetDockerImage() string { +func (x *UpdateRequest) GetContainerImage() string { if x != nil { - return x.DockerImage + return x.ContainerImage } return "" } @@ -337,7 +337,7 @@ var File_api_agent_proto protoreflect.FileDescriptor var file_api_agent_proto_rawDesc = []byte{ 0x0a, 0x0f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x12, 0x05, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x22, 0xfd, 0x01, 0x0a, 0x0d, 0x55, 0x70, 0x64, + 0x6f, 0x12, 0x05, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x22, 0x83, 0x02, 0x0a, 0x0d, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, @@ -348,35 +348,35 @@ var file_api_agent_proto_rawDesc = []byte{ 0x52, 0x08, 0x66, 0x6c, 0x61, 0x67, 0x53, 0x61, 0x66, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x54, 0x4c, 0x53, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x6d, 0x54, 0x4c, 0x53, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x21, 0x0a, - 0x0c, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, - 0x12, 0x26, 0x0a, 0x05, 0x63, 0x65, 0x72, 0x74, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x10, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x65, 0x72, 0x74, 0x73, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x05, 0x63, 0x65, 0x72, 0x74, 0x73, 0x22, 0x66, 0x0a, 0x09, 0x43, 0x65, 0x72, 0x74, - 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x61, 0x5f, 0x63, 0x61, 0x65, 0x72, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x61, 0x43, 0x61, 0x65, 0x72, 0x74, - 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, - 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, - 0x22, 0x22, 0x0a, 0x0e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x03, 0x65, 0x72, 0x72, 0x22, 0x11, 0x0a, 0x0f, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x24, 0x0a, 0x10, 0x52, 0x6f, 0x6c, 0x6c, 0x62, - 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, - 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x72, 0x72, 0x32, 0x7c, 0x0a, - 0x02, 0x4f, 0x53, 0x12, 0x37, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x14, 0x2e, - 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x08, - 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x16, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, - 0x2e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x17, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, - 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x20, 0x5a, 0x1e, 0x6f, - 0x70, 0x65, 0x6e, 0x65, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x6b, 0x75, 0x62, - 0x65, 0x6f, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x27, 0x0a, + 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x63, 0x65, 0x72, 0x74, 0x73, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x65, + 0x72, 0x74, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x63, 0x65, 0x72, 0x74, 0x73, 0x22, 0x66, + 0x0a, 0x09, 0x43, 0x65, 0x72, 0x74, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x63, + 0x61, 0x5f, 0x63, 0x61, 0x65, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, + 0x61, 0x43, 0x61, 0x65, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x22, 0x22, 0x0a, 0x0e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x72, 0x72, 0x22, 0x11, 0x0a, 0x0f, 0x52, 0x6f, + 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x24, 0x0a, + 0x10, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, + 0x65, 0x72, 0x72, 0x32, 0x7c, 0x0a, 0x02, 0x4f, 0x53, 0x12, 0x37, 0x0a, 0x06, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x12, 0x14, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x61, 0x67, 0x65, 0x6e, + 0x74, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x08, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x16, + 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, + 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x42, 0x20, 0x5a, 0x1e, 0x6f, 0x70, 0x65, 0x6e, 0x65, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x6f, + 0x72, 0x67, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x6f, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/cmd/agent/api/agent.proto b/cmd/agent/api/agent.proto index f70cafb5..f1d75229 100644 --- a/cmd/agent/api/agent.proto +++ b/cmd/agent/api/agent.proto @@ -28,7 +28,7 @@ message UpdateRequest { bool flagSafe = 4; bool mTLS = 5; string image_type = 6; - string docker_image = 7; + string container_image = 7; CertsInfo certs = 8; } diff --git a/cmd/agent/server/containerd_image.go b/cmd/agent/server/containerd_image.go new file mode 100644 index 00000000..094985f6 --- /dev/null +++ b/cmd/agent/server/containerd_image.go @@ -0,0 +1,106 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package server implements server of os-agent and listener of os-agent server. The server uses gRPC interface. +package server + +import ( + "bufio" + "io" + "os" + + "github.com/sirupsen/logrus" + + pb "openeuler.org/KubeOS/cmd/agent/api" +) + +var ( + defaultNamespace = "k8s.io" +) + +type conImageHandler struct{} + +func (c conImageHandler) downloadImage(req *pb.UpdateRequest) (string, error) { + neededPath, err := prepareEnv() + if err != nil { + return "", err + } + if _, err = c.getRootfsArchive(req, neededPath); err != nil { + return "", err + } + return createOSImage(neededPath) +} + +func (c conImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath preparePath) (string, error) { + imageName := req.ContainerImage + mountPath := neededPath.mountPath + logrus.Infof("start pull %s", imageName) + + if err := runCommand("crictl", "pull", imageName); err != nil { + return "", err + } + if err := checkAndCleanMount(mountPath); err != nil { + logrus.Errorln("containerd clean environment error", err) + return "", err + } + logrus.Infof("start get rootfs %s", imageName) + if err := runCommand("ctr", "-n="+defaultNamespace, "images", "mount", "--rw", + imageName, mountPath); err != nil { + return "", err + } + defer checkAndCleanMount(mountPath) + if err := copyFile(neededPath.tarPath, mountPath+"/"+rootfsArchive); err != nil { + return "", err + } + return "", nil + return neededPath.tarPath, nil + +} + +func checkAndCleanMount(mountPath string) error { + ctrSnapshotCmd := "ctr " + "-n=" + defaultNamespace + " snapshots ls | grep " + mountPath + " | awk '{print $1}'" + existSnapshot, err := runCommandWithOut("bash", "-c", ctrSnapshotCmd) + if err != nil { + return err + } + if existSnapshot != "" { + if err = runCommand("ctr", "-n="+defaultNamespace, "images", "unmount", mountPath); err != nil { + return err + } + if err = runCommand("ctr", "-n="+defaultNamespace, "snapshots", "delete", mountPath); err != nil { + return err + } + } + return nil +} + +func copyFile(dstFileName string, srcFileName string) error { + srcFile, err := os.Open(srcFileName) + if err != nil { + return err + } + defer srcFile.Close() + + reader := bufio.NewReader(srcFile) + + dstFile, err := os.OpenFile(dstFileName, os.O_WRONLY|os.O_CREATE, imgPermission) + if err != nil { + return err + } + writer := bufio.NewWriter(dstFile) + + defer dstFile.Close() + if _, err = io.Copy(writer, reader); err != nil { + return err + } + return nil +} diff --git a/cmd/agent/server/disk_image.go b/cmd/agent/server/disk_image.go index c876197b..4ffbfaba 100644 --- a/cmd/agent/server/disk_image.go +++ b/cmd/agent/server/disk_image.go @@ -32,6 +32,27 @@ import ( pb "openeuler.org/KubeOS/cmd/agent/api" ) +type diskHandler struct{} + +func (d diskHandler) downloadImage(req *pb.UpdateRequest) (string, error) { + imagePath, err := d.getRootfsArchive(req, preparePath{}) + if err != nil { + return "", err + } + return imagePath, nil +} + +func (d diskHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath preparePath) (string, error) { + imagePath, err := download(req) + if err != nil { + return "", err + } + if err = checkSumMatch(imagePath, req.CheckSum); err != nil { + return "", err + } + return imagePath, nil +} + func download(req *pb.UpdateRequest) (string, error) { resp, err := getImageURL(req) if err != nil { diff --git a/cmd/agent/server/docker_image.go b/cmd/agent/server/docker_image.go index 735ace08..2a52634f 100644 --- a/cmd/agent/server/docker_image.go +++ b/cmd/agent/server/docker_image.go @@ -14,168 +14,54 @@ package server import ( - "context" - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "syscall" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" - "github.com/docker/docker/pkg/archive" "github.com/sirupsen/logrus" pb "openeuler.org/KubeOS/cmd/agent/api" ) -func pullOSImage(req *pb.UpdateRequest) (string, error) { - ctx := context.Background() - cli, err := client.NewEnvClient() - if err != nil { - return "", err - } - imageName := req.DockerImage - logrus.Infof("start pull %s", imageName) - out, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) - if err != nil { - return "", err - } - defer out.Close() - if _, err = ioutil.ReadAll(out); err != nil { - return "", err - } +type dockerImageHandler struct{} - containerName := "kubeos-temp" - containers, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true}) - for _, container := range containers { - if container.Names[0] == "/"+containerName { - if err = cli.ContainerRemove(ctx, container.ID, types.ContainerRemoveOptions{}); err != nil { - return "", err - } - } - } - info, err := cli.ContainerCreate(ctx, &container.Config{ - Image: imageName, - }, nil, nil, containerName) - if err != nil { - return "", err - } - defer cli.ContainerRemove(ctx, info.ID, types.ContainerRemoveOptions{}) - tarStream, stat, err := cli.CopyFromContainer(ctx, info.ID, "/os.tar") +func (d dockerImageHandler) downloadImage(req *pb.UpdateRequest) (string, error) { + neededPath, err := prepareEnv() if err != nil { return "", err } - defer tarStream.Close() - - fs := syscall.Statfs_t{} - if err = syscall.Statfs(PersistDir, &fs); err != nil { - return "", err - } - needGBSize := 3 - kb := 1024 - needDiskSize := needGBSize * kb * kb * kb - if int64(fs.Bfree)*fs.Bsize < int64(needDiskSize) { // these data come from disk size, will not overflow - return "", fmt.Errorf("space is not enough for downloaing") - } - - tmpUpdatePath := filepath.Join(PersistDir, "/KubeOS-Update") - tmpMountPath := filepath.Join(tmpUpdatePath, "/kubeos-update") - tmpTarPath := filepath.Join(tmpUpdatePath, "/os.tar") - imagePath := filepath.Join(PersistDir, "/update.img") - - if err = cleanSpace(tmpUpdatePath, tmpMountPath, imagePath); err != nil { - return "", err - } - if err = os.MkdirAll(tmpMountPath, imgPermission); err != nil { + if _, err = d.getRootfsArchive(req, neededPath); err != nil { return "", err } - defer os.RemoveAll(tmpUpdatePath) + return createOSImage(neededPath) +} - srcInfo := archive.CopyInfo{ - Path: "/", - Exists: true, - IsDir: stat.Mode.IsDir(), - } - if err = archive.CopyTo(tarStream, srcInfo, tmpUpdatePath); err != nil { +func (d dockerImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath preparePath) (string, error) { + imageName := req.ContainerImage + logrus.Infof("start pull %s", imageName) + if err := runCommand("docker", "pull", imageName); err != nil { return "", err } - if err = runCommand("dd", "if=/dev/zero", "of="+imagePath, "bs=2M", "count=1024"); err != nil { + containerName := "kubeos-temp" + dockerPsCmd := "docker ps -a -f=name=" + containerName + "| awk 'NR==2' | awk '{print $1}'" + existId, err := runCommandWithOut("bash", "-c", dockerPsCmd) + if err != nil { return "", err } - if err = os.Chmod(imagePath, imgPermission); err != nil { - return "", err + if existId != "" { + logrus.Infoln("kubeos-temp container exist,start clean environment first") + if err := runCommand("docker", "rm", existId); err != nil { + return "", err + } } - if err = runCommand("mkfs.ext4", "-L", "ROOT-A", imagePath); err != nil { + logrus.Infof("start get rootfs") + containerId, err := runCommandWithOut("docker", "create", "--name", containerName, imageName) + if err != nil { return "", err } - if err = runCommand("mount", "-o", "loop", imagePath, tmpMountPath); err != nil { + if err := runCommand("docker", "cp", containerId+":/"+rootfsArchive, neededPath.updatePath); err != nil { return "", err } defer func() { - syscall.Unmount(tmpMountPath, 0) - runCommand("losetup", "-D") - }() - - logrus.Infoln("downloading to file " + imagePath) - if err = runCommand("tar", "-xvf", tmpTarPath, "-C", tmpMountPath); err != nil { - return "", err - } - return imagePath, nil -} - -func cleanSpace(updatePath, mountPath, imagePath string) error { - isFileExist, err := checkFileExist(mountPath) - if err != nil { - return err - } - if isFileExist { - var st syscall.Stat_t - if err := syscall.Lstat(mountPath, &st); err != nil { - return err - } - dev := st.Dev - parent := filepath.Dir(mountPath) - if err := syscall.Lstat(parent, &st); err != nil { - return err - } - if dev != st.Dev { - if err := syscall.Unmount(mountPath, 0); err != nil { - return err - } + if err := runCommand("docker", "rm", containerId); err != nil { + logrus.Errorln("remove kubeos-temp container error", err) } - } - - if err = deleteFile(updatePath); err != nil { - return err - } - - if err = deleteFile(imagePath); err != nil { - return err - } - return nil -} - -func deleteFile(path string) error { - isFileExist, err := checkFileExist(path) - if err != nil { - return err - } - if isFileExist { - if err = os.RemoveAll(path); err != nil { - return err - } - } - return nil -} -func checkFileExist(path string) (bool, error) { - if _, err := os.Stat(path); err == nil { - return true, nil - } else if errors.Is(err, os.ErrNotExist) { - return false, nil - } else { - return false, err - } + }() + return neededPath.tarPath, nil } diff --git a/cmd/agent/server/server.go b/cmd/agent/server/server.go index e0216a90..67e3f7cf 100644 --- a/cmd/agent/server/server.go +++ b/cmd/agent/server/server.go @@ -104,25 +104,21 @@ func (s *Server) Rollback(_ context.Context, req *pb.RollbackRequest) (*pb.Rollb func (s *Server) update(req *pb.UpdateRequest) error { action := req.ImageType - var imagePath string - var err error + var handler imageDownload switch action { case "docker": - imagePath, err = pullOSImage(req) - if err != nil { - return err - } + handler = dockerImageHandler{} + case "containerd": + handler = conImageHandler{} case "disk": - imagePath, err = download(req) - if err != nil { - return err - } - if err = checkSumMatch(imagePath, req.CheckSum); err != nil { - return err - } + handler = diskHandler{} default: return fmt.Errorf("image type %s cannot be recognized", action) } + imagePath, err := handler.downloadImage(req) + if err != nil { + return err + } side, next, err := getNextPart(partA, partB) if err != nil { return err diff --git a/cmd/agent/server/utils.go b/cmd/agent/server/utils.go index 8fa6554a..a38dee0d 100644 --- a/cmd/agent/server/utils.go +++ b/cmd/agent/server/utils.go @@ -14,14 +14,44 @@ package server import ( + "errors" "fmt" "os" "os/exec" + "path/filepath" "strings" + "syscall" "github.com/sirupsen/logrus" + + pb "openeuler.org/KubeOS/cmd/agent/api" +) + +const ( + needGBSize = 3 // the max size of update files needed + // KB is 1024 B + KB = 1024 +) + +var ( + rootfsArchive = "os.tar" + updateDir = "KubeOS-Update" + mountDir = "kubeos-update" + osImageName = "update.img" ) +type imageDownload interface { + downloadImage(req *pb.UpdateRequest) (string, error) + getRootfsArchive(req *pb.UpdateRequest, neededPath preparePath) (string, error) +} + +type preparePath struct { + updatePath string + mountPath string + tarPath string + imagePath string +} + func runCommand(name string, args ...string) error { out, err := exec.Command(name, args...).CombinedOutput() if err != nil { @@ -30,6 +60,21 @@ func runCommand(name string, args ...string) error { return nil } +func runCommandWithOut(name string, args ...string) (string, error) { + out, err := exec.Command(name, args...).CombinedOutput() + if err != nil { + return "", fmt.Errorf("fail to run command:%s %v out:%s err:%s", name, args, out, err) + } + return deleteNewline(string(out)), nil +} + +func deleteNewline(out string) string { + if strings.HasSuffix(out, "\n") { + out = strings.TrimSuffix(out, "\n") + } + return out +} + func install(imagePath string, side string, next string) error { if err := runCommand("dd", "if="+imagePath, "of="+side, "bs=8M"); err != nil { return err @@ -57,3 +102,133 @@ func getNextPart(partA string, partB string) (string, string, error) { } return side, next, nil } + +func createOSImage(neededPath preparePath) (string, error) { + imagePath := neededPath.imagePath + updatePath := neededPath.updatePath + if err := runCommand("dd", "if=/dev/zero", "of="+imagePath, "bs=2M", "count=1024"); err != nil { + return "", err + } + if err := os.Chmod(imagePath, imgPermission); err != nil { + return "", err + } + if err := runCommand("mkfs.ext4", "-L", "ROOT-A", imagePath); err != nil { + return "", err + } + mountPath := neededPath.mountPath + if err := runCommand("mount", "-o", "loop", imagePath, mountPath); err != nil { + return "", err + } + defer func() { + if err := syscall.Unmount(mountPath, 0); err != nil { + logrus.Errorln("umount error " + mountPath) + } + if err := runCommand("losetup", "-D"); err != nil { + logrus.Errorln("delete loop device error") + } + if err := os.RemoveAll(updatePath); err != nil { + logrus.Errorln("remove dir error " + updatePath) + } + }() + logrus.Infoln("downloading to file " + imagePath) + tarPath := neededPath.tarPath + if err := runCommand("tar", "-xvf", tarPath, "-C", mountPath); err != nil { + return "", err + } + return imagePath, nil +} + +func prepareEnv() (preparePath, error) { + if err := checkDiskSize(needGBSize, PersistDir); err != nil { + return preparePath{}, err + } + updatePath := splicePath(PersistDir, updateDir) + mountPath := splicePath(updatePath, mountDir) + tarPath := splicePath(updatePath, rootfsArchive) + imagePath := splicePath(PersistDir, osImageName) + + if err := cleanSpace(updatePath, mountPath, imagePath); err != nil { + return preparePath{}, err + } + if err := os.MkdirAll(mountPath, imgPermission); err != nil { + return preparePath{}, err + } + upgradePath := preparePath{ + updatePath: updatePath, + mountPath: mountPath, + tarPath: tarPath, + imagePath: imagePath, + } + return upgradePath, nil +} + +func checkDiskSize(needGBSize int, path string) error { + fs := syscall.Statfs_t{} + if err := syscall.Statfs(path, &fs); err != nil { + return err + } + needDiskSize := needGBSize * KB * KB * KB + if int64(fs.Bfree)*fs.Bsize < int64(needDiskSize) { // these data come from disk size, will not overflow + return fmt.Errorf("space is not enough for downloaing") + } + return nil +} + +func splicePath(prefix string, path string) string { + return filepath.Join(prefix, path) +} + +func cleanSpace(updatePath, mountPath, imagePath string) error { + isFileExist, err := checkFileExist(mountPath) + if err != nil { + return err + } + if isFileExist { + var st syscall.Stat_t + if err = syscall.Lstat(mountPath, &st); err != nil { + return err + } + dev := st.Dev + parent := filepath.Dir(mountPath) + if err = syscall.Lstat(parent, &st); err != nil { + return err + } + if dev != st.Dev { + if err = syscall.Unmount(mountPath, 0); err != nil { + return err + } + } + } + + if err = deleteFile(updatePath); err != nil { + return err + } + + if err = deleteFile(imagePath); err != nil { + return err + } + return nil +} + +func deleteFile(path string) error { + isFileExist, err := checkFileExist(path) + if err != nil { + return err + } + if isFileExist { + if err = os.RemoveAll(path); err != nil { + return err + } + } + return nil +} + +func checkFileExist(path string) (bool, error) { + if _, err := os.Stat(path); err == nil { + return true, nil + } else if errors.Is(err, os.ErrNotExist) { + return false, nil + } else { + return false, err + } +} diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index fdd31eaa..30d8ae0f 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -77,12 +77,16 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re osVersionNode := node.Status.NodeInfo.OSImage drainer := &drain.Helper{ - Client: r.kubeclientset, - Force: true, - GracePeriodSeconds: -1, - IgnoreAllDaemonSets: true, - Out: os.Stdout, - ErrOut: os.Stderr, + Ctx: ctx, + Client: r.kubeclientset, + GracePeriodSeconds: -1, + Out: os.Stdout, + ErrOut: os.Stderr, + } + if osInstance.Spec.EvictPodForce { + drainer.DeleteEmptyDirData = true + drainer.IgnoreAllDaemonSets = true + drainer.Force = true } if osVersionNode == osVersionSpec { delete(node.Labels, values.LabelUpgrading) @@ -107,15 +111,15 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re case "upgrade": version := osVersionSpec downloadInfo := &agentclient.DownloadInfo{ - ImageURL: osInstance.Spec.ImageURL, - FlagSafe: osInstance.Spec.FlagSafe, - CheckSum: osInstance.Spec.CheckSum, - CaCert: osInstance.Spec.CaCert, - ClientCert: osInstance.Spec.ClientCert, - ClientKey: osInstance.Spec.ClientKey, - MTLS: osInstance.Spec.MTLS, - ImageType: osInstance.Spec.ImageType, - DockerImage: osInstance.Spec.DockerImage, + ImageURL: osInstance.Spec.ImageURL, + FlagSafe: osInstance.Spec.FlagSafe, + CheckSum: osInstance.Spec.CheckSum, + CaCert: osInstance.Spec.CaCert, + ClientCert: osInstance.Spec.ClientCert, + ClientKey: osInstance.Spec.ClientKey, + MTLS: osInstance.Spec.MTLS, + ImageType: osInstance.Spec.ImageType, + ContainerImage: osInstance.Spec.ContainerImage, } if err := r.Connection.UpdateSpec(version, downloadInfo); err != nil { return values.RequeueNow, err diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index f240b8de..20acaf99 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -40,8 +40,10 @@ spec: type: string clientkey: type: string - dockerimage: + containerimage: type: string + evictpodforce: + type: boolean flagSafe: type: boolean imagetype: @@ -58,7 +60,8 @@ spec: type: string required: - checksum - - dockerimage + - containerimage + - evictpodforce - flagSafe - imagetype - imageurl diff --git a/pkg/agentclient/connection.go b/pkg/agentclient/connection.go index f5e2681b..a72a30de 100644 --- a/pkg/agentclient/connection.go +++ b/pkg/agentclient/connection.go @@ -32,15 +32,15 @@ type Client struct { // DownloadInfo contains the information required for image download type DownloadInfo struct { - ImageURL string - FlagSafe bool - CheckSum string - CaCert string - ClientCert string - ClientKey string - MTLS bool - ImageType string - DockerImage string + ImageURL string + FlagSafe bool + CheckSum string + CaCert string + ClientCert string + ClientKey string + MTLS bool + ImageType string + ContainerImage string } // New create a gRPC channel to communicate with the server and return a client stub to perform RPCs @@ -72,14 +72,14 @@ func (c *Client) UpdateSpec(version string, downloadInfo *DownloadInfo) error { } _, err := c.client.Update(context.Background(), &pb.UpdateRequest{ - Version: version, - ImageUrl: downloadInfo.ImageURL, - FlagSafe: downloadInfo.FlagSafe, - CheckSum: downloadInfo.CheckSum, - MTLS: downloadInfo.MTLS, - Certs: certs, - ImageType: downloadInfo.ImageType, - DockerImage: downloadInfo.DockerImage, + Version: version, + ImageUrl: downloadInfo.ImageURL, + FlagSafe: downloadInfo.FlagSafe, + CheckSum: downloadInfo.CheckSum, + MTLS: downloadInfo.MTLS, + Certs: certs, + ImageType: downloadInfo.ImageType, + ContainerImage: downloadInfo.ContainerImage, }) return err } -- Gitee