diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go index 862d4080018b8587a047468d23ff585385b8382f..2b2554e215516f6caf8457e4b72addf3cbcc8bc6 100644 --- a/api/v1alpha1/os_types.go +++ b/api/v1alpha1/os_types.go @@ -25,11 +25,12 @@ type OSSpec struct { FlagSafe bool `json:"flagSafe"` MTLS bool `json:"mtls"` ImageType string `json:"imagetype"` - DockerImage string `json:"dockerimage"` + ContainerImage string `json:"containerimage"` OpsType string `json:"opstype"` CaCert string `json:"cacert"` ClientCert string `json:"clientcert"` ClientKey string `json:"clientkey"` + EvictPodForce bool `json:"evictpodforce"` } // +kubebuilder:subresource:status diff --git a/cmd/agent/api/agent.pb.go b/cmd/agent/api/agent.pb.go index 9aee75b3de5990e40130146f4794dbb207e0d9ca..336224d9260bb0e89b46e4600d11f280f3700950 100644 --- a/cmd/agent/api/agent.pb.go +++ b/cmd/agent/api/agent.pb.go @@ -11,8 +11,8 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.12.3 +// protoc-gen-go v1.28.1 +// protoc v3.14.0 // source: api/agent.proto package agent @@ -40,14 +40,14 @@ type UpdateRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` - ImageUrl string `protobuf:"bytes,2,opt,name=image_url,json=imageUrl,proto3" json:"image_url,omitempty"` - CheckSum string `protobuf:"bytes,3,opt,name=check_sum,json=checkSum,proto3" json:"check_sum,omitempty"` - FlagSafe bool `protobuf:"varint,4,opt,name=flagSafe,proto3" json:"flagSafe,omitempty"` - MTLS bool `protobuf:"varint,5,opt,name=mTLS,proto3" json:"mTLS,omitempty"` - ImageType string `protobuf:"bytes,6,opt,name=image_type,json=imageType,proto3" json:"image_type,omitempty"` - DockerImage string `protobuf:"bytes,7,opt,name=docker_image,json=dockerImage,proto3" json:"docker_image,omitempty"` - Certs *CertsInfo `protobuf:"bytes,8,opt,name=certs,proto3" json:"certs,omitempty"` + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + ImageUrl string `protobuf:"bytes,2,opt,name=image_url,json=imageUrl,proto3" json:"image_url,omitempty"` + CheckSum string `protobuf:"bytes,3,opt,name=check_sum,json=checkSum,proto3" json:"check_sum,omitempty"` + FlagSafe bool `protobuf:"varint,4,opt,name=flagSafe,proto3" json:"flagSafe,omitempty"` + MTLS bool `protobuf:"varint,5,opt,name=mTLS,proto3" json:"mTLS,omitempty"` + ImageType string `protobuf:"bytes,6,opt,name=image_type,json=imageType,proto3" json:"image_type,omitempty"` + ContainerImage string `protobuf:"bytes,7,opt,name=container_image,json=containerImage,proto3" json:"container_image,omitempty"` + Certs *CertsInfo `protobuf:"bytes,8,opt,name=certs,proto3" json:"certs,omitempty"` } func (x *UpdateRequest) Reset() { @@ -124,9 +124,9 @@ func (x *UpdateRequest) GetImageType() string { return "" } -func (x *UpdateRequest) GetDockerImage() string { +func (x *UpdateRequest) GetContainerImage() string { if x != nil { - return x.DockerImage + return x.ContainerImage } return "" } @@ -337,7 +337,7 @@ var File_api_agent_proto protoreflect.FileDescriptor var file_api_agent_proto_rawDesc = []byte{ 0x0a, 0x0f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x12, 0x05, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x22, 0xfd, 0x01, 0x0a, 0x0d, 0x55, 0x70, 0x64, + 0x6f, 0x12, 0x05, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x22, 0x83, 0x02, 0x0a, 0x0d, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x75, 0x72, @@ -348,35 +348,35 @@ var file_api_agent_proto_rawDesc = []byte{ 0x52, 0x08, 0x66, 0x6c, 0x61, 0x67, 0x53, 0x61, 0x66, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x54, 0x4c, 0x53, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x6d, 0x54, 0x4c, 0x53, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x21, 0x0a, - 0x0c, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x07, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, - 0x12, 0x26, 0x0a, 0x05, 0x63, 0x65, 0x72, 0x74, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x10, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x65, 0x72, 0x74, 0x73, 0x49, 0x6e, 0x66, - 0x6f, 0x52, 0x05, 0x63, 0x65, 0x72, 0x74, 0x73, 0x22, 0x66, 0x0a, 0x09, 0x43, 0x65, 0x72, 0x74, - 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x63, 0x61, 0x5f, 0x63, 0x61, 0x65, 0x72, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x61, 0x43, 0x61, 0x65, 0x72, 0x74, - 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, - 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, - 0x22, 0x22, 0x0a, 0x0e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x03, 0x65, 0x72, 0x72, 0x22, 0x11, 0x0a, 0x0f, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x24, 0x0a, 0x10, 0x52, 0x6f, 0x6c, 0x6c, 0x62, - 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, - 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x72, 0x72, 0x32, 0x7c, 0x0a, - 0x02, 0x4f, 0x53, 0x12, 0x37, 0x0a, 0x06, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x12, 0x14, 0x2e, - 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x55, 0x70, 0x64, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x08, - 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x16, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, - 0x2e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x17, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, - 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x20, 0x5a, 0x1e, 0x6f, - 0x70, 0x65, 0x6e, 0x65, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x6b, 0x75, 0x62, - 0x65, 0x6f, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x27, 0x0a, + 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x6d, 0x61, 0x67, 0x65, + 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x72, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x26, 0x0a, 0x05, 0x63, 0x65, 0x72, 0x74, 0x73, 0x18, + 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x65, + 0x72, 0x74, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x05, 0x63, 0x65, 0x72, 0x74, 0x73, 0x22, 0x66, + 0x0a, 0x09, 0x43, 0x65, 0x72, 0x74, 0x73, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x19, 0x0a, 0x08, 0x63, + 0x61, 0x5f, 0x63, 0x61, 0x65, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, + 0x61, 0x43, 0x61, 0x65, 0x72, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x5f, 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x43, 0x65, 0x72, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x4b, 0x65, 0x79, 0x22, 0x22, 0x0a, 0x0e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x72, 0x72, 0x22, 0x11, 0x0a, 0x0f, 0x52, 0x6f, + 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x24, 0x0a, + 0x10, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x72, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, + 0x65, 0x72, 0x72, 0x32, 0x7c, 0x0a, 0x02, 0x4f, 0x53, 0x12, 0x37, 0x0a, 0x06, 0x55, 0x70, 0x64, + 0x61, 0x74, 0x65, 0x12, 0x14, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x55, 0x70, 0x64, 0x61, + 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x61, 0x67, 0x65, 0x6e, + 0x74, 0x2e, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x3d, 0x0a, 0x08, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x12, 0x16, + 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x2e, 0x52, + 0x6f, 0x6c, 0x6c, 0x62, 0x61, 0x63, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x42, 0x20, 0x5a, 0x1e, 0x6f, 0x70, 0x65, 0x6e, 0x65, 0x75, 0x6c, 0x65, 0x72, 0x2e, 0x6f, + 0x72, 0x67, 0x2f, 0x6b, 0x75, 0x62, 0x65, 0x6f, 0x73, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x61, 0x67, + 0x65, 0x6e, 0x74, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/cmd/agent/api/agent.proto b/cmd/agent/api/agent.proto index f70cafb58f71f8e1ff1170dfd61cd9d75415d709..f1d75229753d37668937975095786163b85109c0 100644 --- a/cmd/agent/api/agent.proto +++ b/cmd/agent/api/agent.proto @@ -28,7 +28,7 @@ message UpdateRequest { bool flagSafe = 4; bool mTLS = 5; string image_type = 6; - string docker_image = 7; + string container_image = 7; CertsInfo certs = 8; } diff --git a/cmd/agent/server/containerd_image.go b/cmd/agent/server/containerd_image.go new file mode 100644 index 0000000000000000000000000000000000000000..094985f635273c5d0a335e29b5d06b00b4029f75 --- /dev/null +++ b/cmd/agent/server/containerd_image.go @@ -0,0 +1,106 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved. + * KubeOS is licensed under the Mulan PSL v2. + * You can use this software according to the terms and conditions of the Mulan PSL v2. + * You may obtain a copy of Mulan PSL v2 at: + * http://license.coscl.org.cn/MulanPSL2 + * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR + * PURPOSE. + * See the Mulan PSL v2 for more details. + */ + +// Package server implements server of os-agent and listener of os-agent server. The server uses gRPC interface. +package server + +import ( + "bufio" + "io" + "os" + + "github.com/sirupsen/logrus" + + pb "openeuler.org/KubeOS/cmd/agent/api" +) + +var ( + defaultNamespace = "k8s.io" +) + +type conImageHandler struct{} + +func (c conImageHandler) downloadImage(req *pb.UpdateRequest) (string, error) { + neededPath, err := prepareEnv() + if err != nil { + return "", err + } + if _, err = c.getRootfsArchive(req, neededPath); err != nil { + return "", err + } + return createOSImage(neededPath) +} + +func (c conImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath preparePath) (string, error) { + imageName := req.ContainerImage + mountPath := neededPath.mountPath + logrus.Infof("start pull %s", imageName) + + if err := runCommand("crictl", "pull", imageName); err != nil { + return "", err + } + if err := checkAndCleanMount(mountPath); err != nil { + logrus.Errorln("containerd clean environment error", err) + return "", err + } + logrus.Infof("start get rootfs %s", imageName) + if err := runCommand("ctr", "-n="+defaultNamespace, "images", "mount", "--rw", + imageName, mountPath); err != nil { + return "", err + } + defer checkAndCleanMount(mountPath) + if err := copyFile(neededPath.tarPath, mountPath+"/"+rootfsArchive); err != nil { + return "", err + } + return "", nil + return neededPath.tarPath, nil + +} + +func checkAndCleanMount(mountPath string) error { + ctrSnapshotCmd := "ctr " + "-n=" + defaultNamespace + " snapshots ls | grep " + mountPath + " | awk '{print $1}'" + existSnapshot, err := runCommandWithOut("bash", "-c", ctrSnapshotCmd) + if err != nil { + return err + } + if existSnapshot != "" { + if err = runCommand("ctr", "-n="+defaultNamespace, "images", "unmount", mountPath); err != nil { + return err + } + if err = runCommand("ctr", "-n="+defaultNamespace, "snapshots", "delete", mountPath); err != nil { + return err + } + } + return nil +} + +func copyFile(dstFileName string, srcFileName string) error { + srcFile, err := os.Open(srcFileName) + if err != nil { + return err + } + defer srcFile.Close() + + reader := bufio.NewReader(srcFile) + + dstFile, err := os.OpenFile(dstFileName, os.O_WRONLY|os.O_CREATE, imgPermission) + if err != nil { + return err + } + writer := bufio.NewWriter(dstFile) + + defer dstFile.Close() + if _, err = io.Copy(writer, reader); err != nil { + return err + } + return nil +} diff --git a/cmd/agent/server/disk_image.go b/cmd/agent/server/disk_image.go index c876197b0a73fddc9c5c361a5b9af6a58099e467..4ffbfaba5c149a12b30527e8f28c0e0313d8b978 100644 --- a/cmd/agent/server/disk_image.go +++ b/cmd/agent/server/disk_image.go @@ -32,6 +32,27 @@ import ( pb "openeuler.org/KubeOS/cmd/agent/api" ) +type diskHandler struct{} + +func (d diskHandler) downloadImage(req *pb.UpdateRequest) (string, error) { + imagePath, err := d.getRootfsArchive(req, preparePath{}) + if err != nil { + return "", err + } + return imagePath, nil +} + +func (d diskHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath preparePath) (string, error) { + imagePath, err := download(req) + if err != nil { + return "", err + } + if err = checkSumMatch(imagePath, req.CheckSum); err != nil { + return "", err + } + return imagePath, nil +} + func download(req *pb.UpdateRequest) (string, error) { resp, err := getImageURL(req) if err != nil { diff --git a/cmd/agent/server/docker_image.go b/cmd/agent/server/docker_image.go index 735ace0834bb2500acb934d8918805809b2a2c20..2a52634f26a26863119080c572c88d38804bc5a9 100644 --- a/cmd/agent/server/docker_image.go +++ b/cmd/agent/server/docker_image.go @@ -14,168 +14,54 @@ package server import ( - "context" - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "syscall" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" - "github.com/docker/docker/pkg/archive" "github.com/sirupsen/logrus" pb "openeuler.org/KubeOS/cmd/agent/api" ) -func pullOSImage(req *pb.UpdateRequest) (string, error) { - ctx := context.Background() - cli, err := client.NewEnvClient() - if err != nil { - return "", err - } - imageName := req.DockerImage - logrus.Infof("start pull %s", imageName) - out, err := cli.ImagePull(ctx, imageName, types.ImagePullOptions{}) - if err != nil { - return "", err - } - defer out.Close() - if _, err = ioutil.ReadAll(out); err != nil { - return "", err - } +type dockerImageHandler struct{} - containerName := "kubeos-temp" - containers, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true}) - for _, container := range containers { - if container.Names[0] == "/"+containerName { - if err = cli.ContainerRemove(ctx, container.ID, types.ContainerRemoveOptions{}); err != nil { - return "", err - } - } - } - info, err := cli.ContainerCreate(ctx, &container.Config{ - Image: imageName, - }, nil, nil, containerName) - if err != nil { - return "", err - } - defer cli.ContainerRemove(ctx, info.ID, types.ContainerRemoveOptions{}) - tarStream, stat, err := cli.CopyFromContainer(ctx, info.ID, "/os.tar") +func (d dockerImageHandler) downloadImage(req *pb.UpdateRequest) (string, error) { + neededPath, err := prepareEnv() if err != nil { return "", err } - defer tarStream.Close() - - fs := syscall.Statfs_t{} - if err = syscall.Statfs(PersistDir, &fs); err != nil { - return "", err - } - needGBSize := 3 - kb := 1024 - needDiskSize := needGBSize * kb * kb * kb - if int64(fs.Bfree)*fs.Bsize < int64(needDiskSize) { // these data come from disk size, will not overflow - return "", fmt.Errorf("space is not enough for downloaing") - } - - tmpUpdatePath := filepath.Join(PersistDir, "/KubeOS-Update") - tmpMountPath := filepath.Join(tmpUpdatePath, "/kubeos-update") - tmpTarPath := filepath.Join(tmpUpdatePath, "/os.tar") - imagePath := filepath.Join(PersistDir, "/update.img") - - if err = cleanSpace(tmpUpdatePath, tmpMountPath, imagePath); err != nil { - return "", err - } - if err = os.MkdirAll(tmpMountPath, imgPermission); err != nil { + if _, err = d.getRootfsArchive(req, neededPath); err != nil { return "", err } - defer os.RemoveAll(tmpUpdatePath) + return createOSImage(neededPath) +} - srcInfo := archive.CopyInfo{ - Path: "/", - Exists: true, - IsDir: stat.Mode.IsDir(), - } - if err = archive.CopyTo(tarStream, srcInfo, tmpUpdatePath); err != nil { +func (d dockerImageHandler) getRootfsArchive(req *pb.UpdateRequest, neededPath preparePath) (string, error) { + imageName := req.ContainerImage + logrus.Infof("start pull %s", imageName) + if err := runCommand("docker", "pull", imageName); err != nil { return "", err } - if err = runCommand("dd", "if=/dev/zero", "of="+imagePath, "bs=2M", "count=1024"); err != nil { + containerName := "kubeos-temp" + dockerPsCmd := "docker ps -a -f=name=" + containerName + "| awk 'NR==2' | awk '{print $1}'" + existId, err := runCommandWithOut("bash", "-c", dockerPsCmd) + if err != nil { return "", err } - if err = os.Chmod(imagePath, imgPermission); err != nil { - return "", err + if existId != "" { + logrus.Infoln("kubeos-temp container exist,start clean environment first") + if err := runCommand("docker", "rm", existId); err != nil { + return "", err + } } - if err = runCommand("mkfs.ext4", "-L", "ROOT-A", imagePath); err != nil { + logrus.Infof("start get rootfs") + containerId, err := runCommandWithOut("docker", "create", "--name", containerName, imageName) + if err != nil { return "", err } - if err = runCommand("mount", "-o", "loop", imagePath, tmpMountPath); err != nil { + if err := runCommand("docker", "cp", containerId+":/"+rootfsArchive, neededPath.updatePath); err != nil { return "", err } defer func() { - syscall.Unmount(tmpMountPath, 0) - runCommand("losetup", "-D") - }() - - logrus.Infoln("downloading to file " + imagePath) - if err = runCommand("tar", "-xvf", tmpTarPath, "-C", tmpMountPath); err != nil { - return "", err - } - return imagePath, nil -} - -func cleanSpace(updatePath, mountPath, imagePath string) error { - isFileExist, err := checkFileExist(mountPath) - if err != nil { - return err - } - if isFileExist { - var st syscall.Stat_t - if err := syscall.Lstat(mountPath, &st); err != nil { - return err - } - dev := st.Dev - parent := filepath.Dir(mountPath) - if err := syscall.Lstat(parent, &st); err != nil { - return err - } - if dev != st.Dev { - if err := syscall.Unmount(mountPath, 0); err != nil { - return err - } + if err := runCommand("docker", "rm", containerId); err != nil { + logrus.Errorln("remove kubeos-temp container error", err) } - } - - if err = deleteFile(updatePath); err != nil { - return err - } - - if err = deleteFile(imagePath); err != nil { - return err - } - return nil -} - -func deleteFile(path string) error { - isFileExist, err := checkFileExist(path) - if err != nil { - return err - } - if isFileExist { - if err = os.RemoveAll(path); err != nil { - return err - } - } - return nil -} -func checkFileExist(path string) (bool, error) { - if _, err := os.Stat(path); err == nil { - return true, nil - } else if errors.Is(err, os.ErrNotExist) { - return false, nil - } else { - return false, err - } + }() + return neededPath.tarPath, nil } diff --git a/cmd/agent/server/server.go b/cmd/agent/server/server.go index e0216a909e84635210581251612cf34dfa63fff1..67e3f7cf55e83f9b73d52f68430d2b16d5318e07 100644 --- a/cmd/agent/server/server.go +++ b/cmd/agent/server/server.go @@ -104,25 +104,21 @@ func (s *Server) Rollback(_ context.Context, req *pb.RollbackRequest) (*pb.Rollb func (s *Server) update(req *pb.UpdateRequest) error { action := req.ImageType - var imagePath string - var err error + var handler imageDownload switch action { case "docker": - imagePath, err = pullOSImage(req) - if err != nil { - return err - } + handler = dockerImageHandler{} + case "containerd": + handler = conImageHandler{} case "disk": - imagePath, err = download(req) - if err != nil { - return err - } - if err = checkSumMatch(imagePath, req.CheckSum); err != nil { - return err - } + handler = diskHandler{} default: return fmt.Errorf("image type %s cannot be recognized", action) } + imagePath, err := handler.downloadImage(req) + if err != nil { + return err + } side, next, err := getNextPart(partA, partB) if err != nil { return err diff --git a/cmd/agent/server/utils.go b/cmd/agent/server/utils.go index 8fa6554ab1ac3772f0006cfcc9705d3a90409923..a38dee0d050506d0a7a15043833059d1079a8683 100644 --- a/cmd/agent/server/utils.go +++ b/cmd/agent/server/utils.go @@ -14,14 +14,44 @@ package server import ( + "errors" "fmt" "os" "os/exec" + "path/filepath" "strings" + "syscall" "github.com/sirupsen/logrus" + + pb "openeuler.org/KubeOS/cmd/agent/api" +) + +const ( + needGBSize = 3 // the max size of update files needed + // KB is 1024 B + KB = 1024 +) + +var ( + rootfsArchive = "os.tar" + updateDir = "KubeOS-Update" + mountDir = "kubeos-update" + osImageName = "update.img" ) +type imageDownload interface { + downloadImage(req *pb.UpdateRequest) (string, error) + getRootfsArchive(req *pb.UpdateRequest, neededPath preparePath) (string, error) +} + +type preparePath struct { + updatePath string + mountPath string + tarPath string + imagePath string +} + func runCommand(name string, args ...string) error { out, err := exec.Command(name, args...).CombinedOutput() if err != nil { @@ -30,6 +60,21 @@ func runCommand(name string, args ...string) error { return nil } +func runCommandWithOut(name string, args ...string) (string, error) { + out, err := exec.Command(name, args...).CombinedOutput() + if err != nil { + return "", fmt.Errorf("fail to run command:%s %v out:%s err:%s", name, args, out, err) + } + return deleteNewline(string(out)), nil +} + +func deleteNewline(out string) string { + if strings.HasSuffix(out, "\n") { + out = strings.TrimSuffix(out, "\n") + } + return out +} + func install(imagePath string, side string, next string) error { if err := runCommand("dd", "if="+imagePath, "of="+side, "bs=8M"); err != nil { return err @@ -57,3 +102,133 @@ func getNextPart(partA string, partB string) (string, string, error) { } return side, next, nil } + +func createOSImage(neededPath preparePath) (string, error) { + imagePath := neededPath.imagePath + updatePath := neededPath.updatePath + if err := runCommand("dd", "if=/dev/zero", "of="+imagePath, "bs=2M", "count=1024"); err != nil { + return "", err + } + if err := os.Chmod(imagePath, imgPermission); err != nil { + return "", err + } + if err := runCommand("mkfs.ext4", "-L", "ROOT-A", imagePath); err != nil { + return "", err + } + mountPath := neededPath.mountPath + if err := runCommand("mount", "-o", "loop", imagePath, mountPath); err != nil { + return "", err + } + defer func() { + if err := syscall.Unmount(mountPath, 0); err != nil { + logrus.Errorln("umount error " + mountPath) + } + if err := runCommand("losetup", "-D"); err != nil { + logrus.Errorln("delete loop device error") + } + if err := os.RemoveAll(updatePath); err != nil { + logrus.Errorln("remove dir error " + updatePath) + } + }() + logrus.Infoln("downloading to file " + imagePath) + tarPath := neededPath.tarPath + if err := runCommand("tar", "-xvf", tarPath, "-C", mountPath); err != nil { + return "", err + } + return imagePath, nil +} + +func prepareEnv() (preparePath, error) { + if err := checkDiskSize(needGBSize, PersistDir); err != nil { + return preparePath{}, err + } + updatePath := splicePath(PersistDir, updateDir) + mountPath := splicePath(updatePath, mountDir) + tarPath := splicePath(updatePath, rootfsArchive) + imagePath := splicePath(PersistDir, osImageName) + + if err := cleanSpace(updatePath, mountPath, imagePath); err != nil { + return preparePath{}, err + } + if err := os.MkdirAll(mountPath, imgPermission); err != nil { + return preparePath{}, err + } + upgradePath := preparePath{ + updatePath: updatePath, + mountPath: mountPath, + tarPath: tarPath, + imagePath: imagePath, + } + return upgradePath, nil +} + +func checkDiskSize(needGBSize int, path string) error { + fs := syscall.Statfs_t{} + if err := syscall.Statfs(path, &fs); err != nil { + return err + } + needDiskSize := needGBSize * KB * KB * KB + if int64(fs.Bfree)*fs.Bsize < int64(needDiskSize) { // these data come from disk size, will not overflow + return fmt.Errorf("space is not enough for downloaing") + } + return nil +} + +func splicePath(prefix string, path string) string { + return filepath.Join(prefix, path) +} + +func cleanSpace(updatePath, mountPath, imagePath string) error { + isFileExist, err := checkFileExist(mountPath) + if err != nil { + return err + } + if isFileExist { + var st syscall.Stat_t + if err = syscall.Lstat(mountPath, &st); err != nil { + return err + } + dev := st.Dev + parent := filepath.Dir(mountPath) + if err = syscall.Lstat(parent, &st); err != nil { + return err + } + if dev != st.Dev { + if err = syscall.Unmount(mountPath, 0); err != nil { + return err + } + } + } + + if err = deleteFile(updatePath); err != nil { + return err + } + + if err = deleteFile(imagePath); err != nil { + return err + } + return nil +} + +func deleteFile(path string) error { + isFileExist, err := checkFileExist(path) + if err != nil { + return err + } + if isFileExist { + if err = os.RemoveAll(path); err != nil { + return err + } + } + return nil +} + +func checkFileExist(path string) (bool, error) { + if _, err := os.Stat(path); err == nil { + return true, nil + } else if errors.Is(err, os.ErrNotExist) { + return false, nil + } else { + return false, err + } +} diff --git a/cmd/proxy/controllers/os_controller.go b/cmd/proxy/controllers/os_controller.go index fdd31eaafbae2c5daadf80c16b692e95ad178d0d..30d8ae0f495c5ff334b4da6ec56b46923c481251 100644 --- a/cmd/proxy/controllers/os_controller.go +++ b/cmd/proxy/controllers/os_controller.go @@ -77,12 +77,16 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re osVersionNode := node.Status.NodeInfo.OSImage drainer := &drain.Helper{ - Client: r.kubeclientset, - Force: true, - GracePeriodSeconds: -1, - IgnoreAllDaemonSets: true, - Out: os.Stdout, - ErrOut: os.Stderr, + Ctx: ctx, + Client: r.kubeclientset, + GracePeriodSeconds: -1, + Out: os.Stdout, + ErrOut: os.Stderr, + } + if osInstance.Spec.EvictPodForce { + drainer.DeleteEmptyDirData = true + drainer.IgnoreAllDaemonSets = true + drainer.Force = true } if osVersionNode == osVersionSpec { delete(node.Labels, values.LabelUpgrading) @@ -107,15 +111,15 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re case "upgrade": version := osVersionSpec downloadInfo := &agentclient.DownloadInfo{ - ImageURL: osInstance.Spec.ImageURL, - FlagSafe: osInstance.Spec.FlagSafe, - CheckSum: osInstance.Spec.CheckSum, - CaCert: osInstance.Spec.CaCert, - ClientCert: osInstance.Spec.ClientCert, - ClientKey: osInstance.Spec.ClientKey, - MTLS: osInstance.Spec.MTLS, - ImageType: osInstance.Spec.ImageType, - DockerImage: osInstance.Spec.DockerImage, + ImageURL: osInstance.Spec.ImageURL, + FlagSafe: osInstance.Spec.FlagSafe, + CheckSum: osInstance.Spec.CheckSum, + CaCert: osInstance.Spec.CaCert, + ClientCert: osInstance.Spec.ClientCert, + ClientKey: osInstance.Spec.ClientKey, + MTLS: osInstance.Spec.MTLS, + ImageType: osInstance.Spec.ImageType, + ContainerImage: osInstance.Spec.ContainerImage, } if err := r.Connection.UpdateSpec(version, downloadInfo); err != nil { return values.RequeueNow, err diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml index f240b8dea84b8dfd97f7f5eede6f174f6f7b16e7..20acaf996d153431258bedfdfd0709e32a3c49b2 100644 --- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml +++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml @@ -40,8 +40,10 @@ spec: type: string clientkey: type: string - dockerimage: + containerimage: type: string + evictpodforce: + type: boolean flagSafe: type: boolean imagetype: @@ -58,7 +60,8 @@ spec: type: string required: - checksum - - dockerimage + - containerimage + - evictpodforce - flagSafe - imagetype - imageurl diff --git a/pkg/agentclient/connection.go b/pkg/agentclient/connection.go index f5e2681bf00713d26f03840f0d75a17007685924..a72a30defdcc0664d0a0ce7a31bbb3916e49c40c 100644 --- a/pkg/agentclient/connection.go +++ b/pkg/agentclient/connection.go @@ -32,15 +32,15 @@ type Client struct { // DownloadInfo contains the information required for image download type DownloadInfo struct { - ImageURL string - FlagSafe bool - CheckSum string - CaCert string - ClientCert string - ClientKey string - MTLS bool - ImageType string - DockerImage string + ImageURL string + FlagSafe bool + CheckSum string + CaCert string + ClientCert string + ClientKey string + MTLS bool + ImageType string + ContainerImage string } // New create a gRPC channel to communicate with the server and return a client stub to perform RPCs @@ -72,14 +72,14 @@ func (c *Client) UpdateSpec(version string, downloadInfo *DownloadInfo) error { } _, err := c.client.Update(context.Background(), &pb.UpdateRequest{ - Version: version, - ImageUrl: downloadInfo.ImageURL, - FlagSafe: downloadInfo.FlagSafe, - CheckSum: downloadInfo.CheckSum, - MTLS: downloadInfo.MTLS, - Certs: certs, - ImageType: downloadInfo.ImageType, - DockerImage: downloadInfo.DockerImage, + Version: version, + ImageUrl: downloadInfo.ImageURL, + FlagSafe: downloadInfo.FlagSafe, + CheckSum: downloadInfo.CheckSum, + MTLS: downloadInfo.MTLS, + Certs: certs, + ImageType: downloadInfo.ImageType, + ContainerImage: downloadInfo.ContainerImage, }) return err }