From fd8a3362ca3d6ecbdb2869bd61a87de84eced2fe Mon Sep 17 00:00:00 2001 From: Yuhang Wei Date: Wed, 29 Nov 2023 10:28:21 +0800 Subject: [PATCH] fix(rust os-agent, proxy): move dd into prepare_upgrade dd update.img to device may takes a while in huawei cloud environment. Considering move dd process into prepare_upgrade stage. The risk is that it damages another partition resulting in unable to rollback to previous version. Signed-off-by: Yuhang Wei --- KubeOS-Rust/agent/src/rpc/agent_impl.rs | 21 +++++++++++-------- KubeOS-Rust/manager/src/api/agent_status.rs | 3 +++ .../manager/src/sys_mgmt/containerd_image.rs | 4 ++-- KubeOS-Rust/manager/src/utils/common.rs | 7 ++++--- .../manager/src/utils/image_manager.rs | 20 ++---------------- .../proxy/src/controller/controller.rs | 21 +++---------------- 6 files changed, 26 insertions(+), 50 deletions(-) diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs index c752ffac..95a28348 100644 --- a/KubeOS-Rust/agent/src/rpc/agent_impl.rs +++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs @@ -22,10 +22,9 @@ use super::{ }; use manager::{ api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest}, - sys_mgmt::{CtrImageHandler, CONFIG_TEMPLATE}, + sys_mgmt::{CtrImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH}, utils::{ clean_env, get_partition_info, switch_boot_menuentry, PreparePath, RealCommandExecutor, - UpgradeImageManager, }, }; @@ -69,7 +68,7 @@ impl AgentImpl { pub fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result { let _lock = self.mutex.lock().unwrap(); debug!("Received an 'prepare upgrade' request: {:?}", req); - info!("Start to upgrade to version: {}", req.version); + info!("Start preparing for upgrading to version: {}", req.version); let handler: Box> = match req.image_type.as_str() { "containerd" => Box::new(ImageType::Containerd(CtrImageHandler::default())), @@ -81,6 +80,7 @@ impl AgentImpl { "Ready to install image: {:?}", image_manager.paths.image_path.display() ); + image_manager.install()?; Ok(Response { status: AgentStatus::UpgradeReady, @@ -92,12 +92,15 @@ impl AgentImpl { info!("Start to upgrade"); let command_executor = RealCommandExecutor {}; let (_, next_partition_info) = get_partition_info(&command_executor)?; - let image_manager = UpgradeImageManager::new( - PreparePath::default(), - next_partition_info, - command_executor, + + // based on boot mode use different command to switch boot partition + let device = next_partition_info.device.as_str(); + let menuentry = next_partition_info.menuentry.as_str(); + switch_boot_menuentry(&command_executor, DEFAULT_GRUBENV_PATH, menuentry)?; + info!( + "Switch to boot partition: {}, device: {}", + menuentry, device ); - image_manager.install()?; self.reboot()?; Ok(Response { status: AgentStatus::Upgraded, @@ -110,7 +113,7 @@ impl AgentImpl { let paths = PreparePath::default(); clean_env(paths.update_path, paths.mount_path, paths.image_path)?; Ok(Response { - status: AgentStatus::NotApplied, + status: AgentStatus::CleanedUp, }) } diff --git a/KubeOS-Rust/manager/src/api/agent_status.rs b/KubeOS-Rust/manager/src/api/agent_status.rs index 7a2edf7d..c3152257 100644 --- a/KubeOS-Rust/manager/src/api/agent_status.rs +++ b/KubeOS-Rust/manager/src/api/agent_status.rs @@ -18,6 +18,7 @@ const AGENT_STATUS_UPGRADEREADY: &str = "UPGRADE-READY"; const AGENT_STATUS_UPGRADED: &str = "UPGRADED"; const AGENT_STATUS_ROLLBACKED: &str = "ROLLBACKED"; const AGENT_STATUS_CONFIGURED: &str = "CONFIGURED"; +const AGENT_STATUS_CLEANEDUP: &str = "CLEANEDUP"; #[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] pub enum AgentStatus { @@ -27,6 +28,7 @@ pub enum AgentStatus { Upgraded, Rollbacked, Configured, + CleanedUp, } impl Default for AgentStatus { @@ -44,6 +46,7 @@ impl std::fmt::Display for AgentStatus { AgentStatus::Upgraded => AGENT_STATUS_UPGRADED, AgentStatus::Rollbacked => AGENT_STATUS_ROLLBACKED, AgentStatus::Configured => AGENT_STATUS_CONFIGURED, + AgentStatus::CleanedUp => AGENT_STATUS_CLEANEDUP, }) } } diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs index b4bdd2c8..479740e4 100644 --- a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs +++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs @@ -60,7 +60,7 @@ impl CtrImageHandler { fn get_image(&self, req: &UpgradeRequest) -> Result<()> { let image_name = &req.container_image; is_valid_image_name(image_name)?; - info!("Start pull image {}", image_name); + info!("Start pulling image {}", image_name); let containerd_command: String; if is_command_available("crictl", &self.executor) { containerd_command = "crictl".to_string(); @@ -68,7 +68,7 @@ impl CtrImageHandler { containerd_command = "ctr".to_string(); } pull_image(&containerd_command, image_name, &self.executor)?; - info!("Start check image digest"); + info!("Start checking image digest"); check_oci_image_digest_match( &containerd_command, image_name, diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs index cc826dd4..7bb75a84 100644 --- a/KubeOS-Rust/manager/src/utils/common.rs +++ b/KubeOS-Rust/manager/src/utils/common.rs @@ -18,7 +18,7 @@ use std::{ }; use anyhow::{anyhow, Result}; -use log::{debug, info}; +use log::{debug, info, trace}; use nix::{mount, mount::MntFlags}; use super::executor::CommandExecutor; @@ -72,7 +72,7 @@ pub fn perpare_env( } pub fn check_disk_size(need_gb: i64, path: &str) -> Result<()> { - info!("Check if there is enough disk space to upgrade"); + trace!("Check if there is enough disk space to upgrade"); let kb = 1024; let fs_stat = nix::sys::statfs::statfs(path)?; let need_disk_size = need_gb * kb * kb * kb; @@ -81,6 +81,7 @@ pub fn check_disk_size(need_gb: i64, path: &str) -> Result<()> { if available_space < need_disk_size { return Err(anyhow!("Space is not enough for downloading")); } + debug!("There is enough disk space to upgrade"); Ok(()) } @@ -89,7 +90,7 @@ pub fn clean_env

(update_path: P, mount_path: P, image_path: P) -> Result<()> where P: AsRef, { - info!("Clean upgrade environment"); + info!("Clean up the residual upgrade environment"); if is_mounted(&mount_path)? { debug!("Umount {}", mount_path.as_ref().display()); if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) { diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs index 42b5a745..ca48712f 100644 --- a/KubeOS-Rust/manager/src/utils/image_manager.rs +++ b/KubeOS-Rust/manager/src/utils/image_manager.rs @@ -25,7 +25,6 @@ use super::{ executor::CommandExecutor, partition::PartitionInfo, }; -use crate::{sys_mgmt::DEFAULT_GRUBENV_PATH, utils::switch_boot_menuentry}; pub struct UpgradeImageManager { pub paths: PreparePath, @@ -129,7 +128,6 @@ impl UpgradeImageManager { pub fn install(&self) -> Result<()> { let image_str = self.image_path_str()?; let device = self.next_partition.device.as_str(); - let menuentry = self.next_partition.menuentry.as_str(); self.executor.run_command( "dd", &[ @@ -139,14 +137,11 @@ impl UpgradeImageManager { ], )?; debug!("Install image {} to {} done", image_str, device); - // based on boot mode use different command to switch boot partition - switch_boot_menuentry(&self.executor, DEFAULT_GRUBENV_PATH, menuentry)?; info!( - "Switch to boot partition: {}, device: {}", - menuentry, device + "Device {} is overwritten and unable to rollback to the previous version anymore if the eviction of node fails", + device ); delete_file_or_dir(image_str)?; - debug!("Remove image {}", image_str); Ok(()) } } @@ -224,17 +219,6 @@ mod tests { .times(1) // Expect it to be called once .returning(|_, _| Ok(())); - //mock install->grub2-set-default - mock.expect_run_command() - .withf(|name, args| { - name == "grub2-editenv" - && args[0] == "/boot/efi/EFI/openEuler/grubenv" - && args[1] == "set" - && args[2] == "saved_entry=B" - }) - .times(1) // Expect it to be called once - .returning(|_, _| Ok(())); - let img_manager = UpgradeImageManager::new( PreparePath { update_path: tmp_dir.into(), diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs index 3a0f1a7c..6410cce1 100644 --- a/KubeOS-Rust/proxy/src/controller/controller.rs +++ b/KubeOS-Rust/proxy/src/controller/controller.rs @@ -25,7 +25,7 @@ use anyhow::Result; use cli::{ client::Client as AgentClient, method::{ - callable_method::RpcMethod, cleanup::CleanupMethod, configure::ConfigureMethod, + callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod, rollback::RollbackMethod, upgrade::UpgradeMethod, }, }; @@ -348,23 +348,8 @@ impl ProxyController { return Err(Error::AgentError { source: e }); } } - match self - .evict_node(&node.name(), os_cr.spec.evictpodforce) - .await - { - Ok(()) => {} - Err(e) => { - match CleanupMethod::new().call(&self.agent_client) { - Ok(_resp) => {} - Err(agent_error) => { - return Err(Error::AgentError { - source: agent_error, - }); - } - } - return Err(e); - } - } + self.evict_node(&node.name(), os_cr.spec.evictpodforce) + .await?; match UpgradeMethod::new().call(&self.agent_client) { Ok(_resp) => {} Err(e) => { -- Gitee