diff --git a/.gitignore b/.gitignore
index 5e56e040ec0902e58df8573adaec65c5da6e9304..0cac753eeb1da6376bc189165bca0c1a59c09325 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1,11 @@
+# vscode settings
+.vscode
+
+# rust dependencies
+target/
+
+# KubeOS bin
 /bin
+
+# scripts-auto
+/scripts-auto
diff --git a/KubeOS-Rust/Cargo.lock b/KubeOS-Rust/Cargo.lock
new file mode 100644
index 0000000000000000000000000000000000000000..89e56ec23bc8b36b99f1d830760f959db2c9ac81
--- /dev/null
+++ b/KubeOS-Rust/Cargo.lock
@@ -0,0 +1,3098 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "aho-corasick"
+version = "0.7.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc936419f96fa211c1b9166887b38e5e40b19958e5b895be7c1f93adec7071ac"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "android-tzdata"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
+
+[[package]]
+name = "android_system_properties"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.75"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6"
+
+[[package]]
+name = "assert-json-diff"
+version = "2.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12"
+dependencies = [
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "async-stream"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51"
+dependencies = [
+ "async-stream-impl",
+ "futures-core",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "async-stream-impl"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "async-trait"
+version = "0.1.74"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "atty"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
+dependencies = [
+ "hermit-abi 0.1.19",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "backoff"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b62ddb9cb1ec0a098ad4bbf9344d0713fa193ae1a80af55febcff2627b6a00c1"
+dependencies = [
+ "getrandom 0.2.10",
+ "instant",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
+[[package]]
+name = "base64"
+version = "0.21.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35636a1494ede3b646cc98f74f8e62c773a38a659ebc777a2cf26b9b74171df9"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bitflags"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635"
+
+[[package]]
+name = "block-buffer"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
+dependencies = [
+ "generic-array",
+]
+
+[[package]]
+name = "bstr"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3569f383e8f1598449f1a423e72e99569137b47740b1da11ef19af3d5c3223"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "bumpalo"
+version = "3.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec"
+
+[[package]]
+name = "bytes"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223"
+
+[[package]]
+name = "cc"
+version = "1.0.83"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "chrono"
+version = "0.4.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38"
+dependencies = [
+ "android-tzdata",
+ "iana-time-zone",
+ "js-sys",
+ "num-traits",
+ "serde",
+ "wasm-bindgen",
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "clap"
+version = "3.2.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5"
+dependencies = [
+ "atty",
+ "bitflags 1.3.2",
+ "clap_derive",
+ "clap_lex",
+ "indexmap 1.9.3",
+ "once_cell",
+ "strsim",
+ "termcolor",
+ "textwrap",
+]
+
+[[package]]
+name = "clap_derive"
+version = "3.2.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65"
+dependencies = [
+ "heck",
+ "proc-macro-error",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2850f2f5a82cbf437dd5af4d49848fbdfc27c157c3d010345776f952765261c5"
+dependencies = [
+ "os_str_bytes",
+]
+
+[[package]]
+name = "cli"
+version = "1.0.6"
+dependencies = [
+ "anyhow",
+ "jsonrpc",
+ "log",
+ "manager",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "core-foundation"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa"
+
+[[package]]
+name = "cpufeatures"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80"
+
+[[package]]
+name = "crypto-common"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
+dependencies = [
+ "generic-array",
+ "typenum",
+]
+
+[[package]]
+name = "darling"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "dashmap"
+version = "4.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
+dependencies = [
+ "cfg-if",
+ "num_cpus",
+]
+
+[[package]]
+name = "derivative"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "difflib"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8"
+
+[[package]]
+name = "digest"
+version = "0.10.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
+dependencies = [
+ "block-buffer",
+ "crypto-common",
+]
+
+[[package]]
+name = "dirs-next"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
+dependencies = [
+ "cfg-if",
+ "dirs-sys-next",
+]
+
+[[package]]
+name = "dirs-sys-next"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
+dependencies = [
+ "libc",
+ "redox_users",
+ "winapi",
+]
+
+[[package]]
+name = "downcast"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1"
+
+[[package]]
+name = "dyn-clone"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "545b22097d44f8a9581187cdf93de7a71e4722bf51200cfaba810865b49a495d"
+
+[[package]]
+name = "either"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07"
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "env_logger"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a12e6657c4c97ebab115a42dcee77225f7f482cdd841cf7088c657a42e9e00e7"
+dependencies = [
+ "atty",
+ "humantime",
+ "log",
+ "regex",
+ "termcolor",
+]
+
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
+[[package]]
+name = "errno"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "fastrand"
+version = "1.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be"
+dependencies = [
+ "instant",
+]
+
+[[package]]
+name = "float-cmp"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
+[[package]]
+name = "form_urlencoded"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652"
+dependencies = [
+ "percent-encoding",
+]
+
+[[package]]
+name = "fragile"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa"
+
+[[package]]
+name = "futures"
+version = "0.3.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817"
+
+[[package]]
+name = "futures-task"
+version = "0.3.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2"
+
+[[package]]
+name = "futures-util"
+version = "0.3.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+]
+
+[[package]]
+name = "generic-array"
+version = "0.14.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
+dependencies = [
+ "typenum",
+ "version_check",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+]
+
+[[package]]
+name = "globset"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a1e17342619edbc21a964c2afbeb6c820c6a2560032872f397bb97ea127bd0a"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "fnv",
+ "log",
+ "regex",
+]
+
+[[package]]
+name = "h2"
+version = "0.3.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d"
+dependencies = [
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "futures-util",
+ "http 0.2.9",
+ "indexmap 1.9.3",
+ "slab",
+ "tokio",
+ "tokio-util 0.7.2",
+ "tracing",
+]
+
+[[package]]
+name = "h2"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "51ee2dd2e4f378392eeff5d51618cd9a63166a2513846bbc55f21cfacd9199d4"
+dependencies = [
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "futures-util",
+ "http 1.1.0",
+ "indexmap 2.2.6",
+ "slab",
+ "tokio",
+ "tokio-util 0.7.2",
+ "tracing",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "hashbrown"
+version = "0.14.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
+
+[[package]]
+name = "heck"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+
+[[package]]
+name = "hermit-abi"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d77f7ec81a6d05a3abb01ab6eb7590f6083d08449fe5a1c8b1e620283546ccb7"
+
+[[package]]
+name = "http"
+version = "0.2.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
+[[package]]
+name = "http"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
+[[package]]
+name = "http-body"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
+dependencies = [
+ "bytes",
+ "http 0.2.9",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "http-body"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643"
+dependencies = [
+ "bytes",
+ "http 1.1.0",
+]
+
+[[package]]
+name = "http-body-util"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "http 1.1.0",
+ "http-body 1.0.0",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "http-range-header"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f"
+
+[[package]]
+name = "httparse"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904"
+
+[[package]]
+name = "httpdate"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+
+[[package]]
+name = "humantime"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
+
+[[package]]
+name = "hyper"
+version = "0.14.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc5e554ff619822309ffd57d8734d77cd5ce6238bc956f037ea06c58238c9899"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "http 0.2.9",
+ "http-body 0.4.5",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2 0.4.9",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "h2 0.4.3",
+ "http 1.1.0",
+ "http-body 1.0.0",
+ "httparse",
+ "itoa",
+ "pin-project-lite",
+ "smallvec",
+ "tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a0bea761b46ae2b24eb4aef630d8d1c398157b6fc29e6350ecf090a0b70c952c"
+dependencies = [
+ "futures-util",
+ "http 1.1.0",
+ "hyper 1.2.0",
+ "hyper-util",
+ "rustls",
+ "rustls-pki-types",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
+]
+
+[[package]]
+name = "hyper-timeout"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1"
+dependencies = [
+ "hyper 0.14.25",
+ "pin-project-lite",
+ "tokio",
+ "tokio-io-timeout",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper 0.14.25",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
+dependencies = [
+ "bytes",
+ "http-body-util",
+ "hyper 1.2.0",
+ "hyper-util",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+ "tower-service",
+]
+
+[[package]]
+name = "hyper-util"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-util",
+ "http 1.1.0",
+ "http-body 1.0.0",
+ "hyper 1.2.0",
+ "pin-project-lite",
+ "socket2 0.5.6",
+ "tokio",
+ "tower",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "iana-time-zone"
+version = "0.1.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8326b86b6cff230b97d0d312a6c40a60726df3332e721f72a1b035f451663b20"
+dependencies = [
+ "android_system_properties",
+ "core-foundation-sys",
+ "iana-time-zone-haiku",
+ "js-sys",
+ "wasm-bindgen",
+ "windows-core",
+]
+
+[[package]]
+name = "iana-time-zone-haiku"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
+[[package]]
+name = "idna"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.3",
+]
+
+[[package]]
+name = "instant"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "io-lifetimes"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2"
+dependencies = [
+ "hermit-abi 0.3.3",
+ "libc",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "ipnet"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+
+[[package]]
+name = "itertools"
+version = "0.10.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38"
+
+[[package]]
+name = "js-sys"
+version = "0.3.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54c0c35952f67de54bb584e9fd912b3023117cbafc0a77d8f3dee1fb5f572fe8"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "json-patch"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb3fa5a61630976fc4c353c70297f2e93f1930e3ccee574d59d618ccbd5154ce"
+dependencies = [
+ "serde",
+ "serde_json",
+ "treediff",
+]
+
+[[package]]
+name = "jsonpath_lib"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eaa63191d68230cccb81c5aa23abd53ed64d83337cacbb25a7b8c7979523774f"
+dependencies = [
+ "log",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "jsonrpc"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fd8d6b3f301ba426b30feca834a2a18d48d5b54e5065496b5c1b05537bee3639"
+dependencies = [
+ "base64 0.13.1",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "jsonrpc-core"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14f7f76aef2d054868398427f6c54943cf3d1caa9a7ec7d0c38d69df97a965eb"
+dependencies = [
+ "futures",
+ "futures-executor",
+ "futures-util",
+ "log",
+ "serde",
+ "serde_derive",
+ "serde_json",
+]
+
+[[package]]
+name = "jsonrpc-derive"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b939a78fa820cdfcb7ee7484466746a7377760970f6f9c6fe19f9edcc8a38d2"
+dependencies = [
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "jsonrpc-ipc-server"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "382bb0206323ca7cda3dcd7e245cea86d37d02457a02a975e3378fb149a48845"
+dependencies = [
+ "futures",
+ "jsonrpc-core",
+ "jsonrpc-server-utils",
+ "log",
+ "parity-tokio-ipc",
+ "parking_lot",
+ "tower-service",
+]
+
+[[package]]
+name = "jsonrpc-server-utils"
+version = "18.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4fdea130485b572c39a460d50888beb00afb3e35de23ccd7fad8ff19f0e0d4"
+dependencies = [
+ "bytes",
+ "futures",
+ "globset",
+ "jsonrpc-core",
+ "lazy_static",
+ "log",
+ "tokio",
+ "tokio-stream",
+ "tokio-util 0.6.10",
+ "unicase",
+]
+
+[[package]]
+name = "k8s-openapi"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4f8de9873b904e74b3533f77493731ee26742418077503683db44e1b3c54aa5c"
+dependencies = [
+ "base64 0.13.1",
+ "bytes",
+ "chrono",
+ "http 0.2.9",
+ "percent-encoding",
+ "serde",
+ "serde-value",
+ "serde_json",
+ "url",
+]
+
+[[package]]
+name = "kbimg"
+version = "1.0.6"
+dependencies = [
+ "anyhow",
+ "clap",
+ "env_logger",
+ "log",
+ "regex",
+ "serde",
+ "sysinfo",
+ "toml 0.7.6",
+]
+
+[[package]]
+name = "kube"
+version = "0.66.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4b96944d327b752df4f62f3a31d8694892af06fb585747c0b5e664927823d1a"
+dependencies = [
+ "k8s-openapi",
+ "kube-client",
+ "kube-core",
+ "kube-derive",
+ "kube-runtime",
+]
+
+[[package]]
+name = "kube-client"
+version = "0.66.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "232db1af3d3680f9289cf0b4db51b2b9fee22550fc65d25869e39b23e0aaa696"
+dependencies = [
+ "base64 0.13.1",
+ "bytes",
+ "chrono",
+ "dirs-next",
+ "either",
+ "futures",
+ "http 0.2.9",
+ "http-body 0.4.5",
+ "hyper 0.14.25",
+ "hyper-timeout",
+ "hyper-tls 0.5.0",
+ "jsonpath_lib",
+ "k8s-openapi",
+ "kube-core",
+ "openssl",
+ "pem",
+ "pin-project",
+ "secrecy",
+ "serde",
+ "serde_json",
+ "serde_yaml",
+ "thiserror",
+ "tokio",
+ "tokio-native-tls",
+ "tokio-util 0.6.10",
+ "tower",
+ "tower-http",
+ "tracing",
+]
+
+[[package]]
+name = "kube-core"
+version = "0.66.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de491f8c9ee97117e0b47a629753e939c2392d5d0a40f6928e582a5fba328098"
+dependencies = [
+ "chrono",
+ "form_urlencoded",
+ "http 0.2.9",
+ "json-patch",
+ "k8s-openapi",
+ "once_cell",
+ "schemars",
+ "serde",
+ "serde_json",
+ "thiserror",
+]
+
+[[package]]
+name = "kube-derive"
+version = "0.66.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fcbb86bb3607245a67c8ad3a52aff41108f36b0d1e9e3e82ffb5760bfd84b965"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "serde_json",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "kube-runtime"
+version = "0.66.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "710729592eb30219b4e84898e91dc991fe09ccafe2c17fec4e45c3426c61abe0"
+dependencies = [
+ "backoff",
+ "dashmap",
+ "derivative",
+ "futures",
+ "json-patch",
+ "k8s-openapi",
+ "kube-client",
+ "pin-project",
+ "serde",
+ "serde_json",
+ "smallvec",
+ "thiserror",
+ "tokio",
+ "tokio-util 0.6.10",
+ "tracing",
+]
+
+[[package]]
+name = "lazy_static"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
+
+[[package]]
+name = "libc"
+version = "0.2.151"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4"
+
+[[package]]
+name = "libredox"
+version = "0.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8"
+dependencies = [
+ "bitflags 2.4.0",
+ "libc",
+ "redox_syscall 0.4.1",
+]
+
+[[package]]
+name = "linked-hash-map"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.3.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef53942eb7bf7ff43a617b3e2c1c4a5ecf5944a7c1bc12d7ee39bbb15e5c1519"
+
+[[package]]
+name = "lock_api"
+version = "0.4.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c4dcd960cc540667f619483fc99102f88d6118b87730e24e8fbe8054b7445e4"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "manager"
+version = "1.0.6"
+dependencies = [
+ "anyhow",
+ "env_logger",
+ "lazy_static",
+ "log",
+ "mockall",
+ "mockito",
+ "nix",
+ "predicates",
+ "regex",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "sha2",
+ "tempfile",
+]
+
+[[package]]
+name = "memchr"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
+
+[[package]]
+name = "memoffset"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "mime"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+
+[[package]]
+name = "mio"
+version = "0.8.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f3d0b296e374a4e6f3c7b0a1f5a51d748a0d34c85e7dc48fc3fa9a87657fe09"
+dependencies = [
+ "libc",
+ "log",
+ "wasi 0.11.0+wasi-snapshot-preview1",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "mockall"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "50e4a1c770583dac7ab5e2f6c139153b783a53a1bbee9729613f193e59828326"
+dependencies = [
+ "cfg-if",
+ "downcast",
+ "fragile",
+ "lazy_static",
+ "mockall_derive",
+ "predicates",
+ "predicates-tree",
+]
+
+[[package]]
+name = "mockall_derive"
+version = "0.11.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "832663583d5fa284ca8810bf7015e46c9fff9622d3cf34bd1eea5003fec06dd0"
+dependencies = [
+ "cfg-if",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "mockito"
+version = "0.31.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "80f9fece9bd97ab74339fe19f4bcaf52b76dcc18e5364c7977c1838f76b38de9"
+dependencies = [
+ "assert-json-diff",
+ "httparse",
+ "lazy_static",
+ "log",
+ "rand 0.8.5",
+ "regex",
+ "serde_json",
+ "serde_urlencoded",
+ "similar",
+]
+
+[[package]]
+name = "native-tls"
+version = "0.2.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
+[[package]]
+name = "nix"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "598beaf3cc6fdd9a5dfb1630c2800c7acd31df7aaf0f565796fba2b53ca1af1b"
+dependencies = [
+ "bitflags 1.3.2",
+ "cfg-if",
+ "libc",
+ "memoffset",
+ "pin-utils",
+]
+
+[[package]]
+name = "normalize-line-endings"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be"
+
+[[package]]
+name = "ntapi"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8a3895c6391c39d7fe7ebc444a87eb2991b2a0bc718fdabd071eec617fc68e4"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "num-traits"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "num_cpus"
+version = "1.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
+dependencies = [
+ "hermit-abi 0.3.3",
+ "libc",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9670a07f94779e00908f3e686eab508878ebb390ba6e604d3a284c00e8d0487b"
+
+[[package]]
+name = "openssl"
+version = "0.10.59"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a257ad03cd8fb16ad4172fedf8094451e1af1c4b70097636ef2eac9a5f0cc33"
+dependencies = [
+ "bitflags 2.4.0",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.95"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40a4130519a360279579c2053038317e40eff64d13fd3f004f9e1b72b8a6aaf9"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "operator"
+version = "1.0.6"
+dependencies = [
+ "anyhow",
+ "assert-json-diff",
+ "async-trait",
+ "cli",
+ "env_logger",
+ "futures",
+ "h2 0.3.16",
+ "http 0.2.9",
+ "hyper 0.14.25",
+ "k8s-openapi",
+ "kube",
+ "log",
+ "manager",
+ "mockall",
+ "regex",
+ "reqwest",
+ "schemars",
+ "serde",
+ "serde_json",
+ "socket2 0.4.9",
+ "thiserror",
+ "thread_local",
+ "tokio",
+ "tokio-retry",
+ "tower-test",
+]
+
+[[package]]
+name = "ordered-float"
+version = "2.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c"
+dependencies = [
+ "num-traits",
+]
+
+[[package]]
+name = "os-agent"
+version = "1.0.6"
+dependencies = [
+ "anyhow",
+ "env_logger",
+ "jsonrpc-core",
+ "jsonrpc-derive",
+ "jsonrpc-ipc-server",
+ "lazy_static",
+ "log",
+ "manager",
+ "nix",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "os_str_bytes"
+version = "6.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1"
+
+[[package]]
+name = "parity-tokio-ipc"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9981e32fb75e004cc148f5fb70342f393830e0a4aa62e3cc93b50976218d42b6"
+dependencies = [
+ "futures",
+ "libc",
+ "log",
+ "rand 0.7.3",
+ "tokio",
+ "winapi",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
+dependencies = [
+ "instant",
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
+dependencies = [
+ "cfg-if",
+ "instant",
+ "libc",
+ "redox_syscall 0.2.16",
+ "smallvec",
+ "winapi",
+]
+
+[[package]]
+name = "pem"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8835c273a76a90455d7344889b0964598e3316e2a79ede8e36f16bdcf2228b8"
+dependencies = [
+ "base64 0.13.1",
+]
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
+
+[[package]]
+name = "pin-project"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422"
+dependencies = [
+ "pin-project-internal",
+]
+
+[[package]]
+name = "pin-project-internal"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58"
+
+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+
+[[package]]
+name = "predicates"
+version = "2.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bc3d91237f5de3bcd9d927e24d03b495adb6135097b001cea7403e2d573d00a9"
+dependencies = [
+ "difflib",
+ "float-cmp",
+ "itertools",
+ "normalize-line-endings",
+ "predicates-core",
+ "regex",
+]
+
+[[package]]
+name = "predicates-core"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da1c2388b1513e1b605fcec39a95e0a9e8ef088f71443ef37099fa9ae6673fcb"
+
+[[package]]
+name = "predicates-tree"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4d86de6de25020a36c6d3643a86d9a6a9f552107c0559c60ea03551b5e16c032"
+dependencies = [
+ "predicates-core",
+ "termtree",
+]
+
+[[package]]
+name = "proc-macro-crate"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d6ea3c4595b96363c13943497db34af4460fb474a95c43f4446ad341b8c9785"
+dependencies = [
+ "toml 0.5.11",
+]
+
+[[package]]
+name = "proc-macro-error"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c"
+dependencies = [
+ "proc-macro-error-attr",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro-error-attr"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "version_check",
+]
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.67"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d433d9f1a3e8c1263d9456598b16fec66f4acc9a74dacffd35c7bb09b3a1328"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "proxy"
+version = "1.0.6"
+dependencies = [
+ "anyhow",
+ "assert-json-diff",
+ "async-trait",
+ "cli",
+ "env_logger",
+ "futures",
+ "h2 0.3.16",
+ "http 0.2.9",
+ "hyper 0.14.25",
+ "k8s-openapi",
+ "kube",
+ "log",
+ "manager",
+ "mockall",
+ "regex",
+ "reqwest",
+ "schemars",
+ "serde",
+ "serde_json",
+ "socket2 0.4.9",
+ "thiserror",
+ "thread_local",
+ "tokio",
+ "tokio-retry",
+ "tower-test",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom 0.2.10",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "rayon"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.2.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
+dependencies = [
+ "bitflags 1.3.2",
+]
+
+[[package]]
+name = "redox_users"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4"
+dependencies = [
+ "getrandom 0.2.10",
+ "libredox",
+ "thiserror",
+]
+
+[[package]]
+name = "regex"
+version = "1.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
+dependencies = [
+ "aho-corasick",
+ "memchr",
+ "regex-syntax",
+]
+
+[[package]]
+name = "regex-syntax"
+version = "0.6.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
+
+[[package]]
+name = "reqwest"
+version = "0.12.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d66674f2b6fb864665eea7a3c1ac4e3dfacd2fda83cf6f935a612e01b0e3338"
+dependencies = [
+ "base64 0.21.5",
+ "bytes",
+ "encoding_rs",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2 0.4.3",
+ "http 1.1.0",
+ "http-body 1.0.0",
+ "http-body-util",
+ "hyper 1.2.0",
+ "hyper-rustls",
+ "hyper-tls 0.6.0",
+ "hyper-util",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustls",
+ "rustls-pemfile",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sync_wrapper",
+ "system-configuration",
+ "tokio",
+ "tokio-native-tls",
+ "tokio-rustls",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "webpki-roots",
+ "winreg",
+]
+
+[[package]]
+name = "ring"
+version = "0.17.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+dependencies = [
+ "cc",
+ "cfg-if",
+ "getrandom 0.2.10",
+ "libc",
+ "spin",
+ "untrusted",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rustix"
+version = "0.37.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fea8ca367a3a01fe35e6943c400addf443c0f57670e6ec51196f71a4b8762dd2"
+dependencies = [
+ "bitflags 1.3.2",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "rustls"
+version = "0.22.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "99008d7ad0bbbea527ec27bddbc0e432c5b87d8175178cee68d2eec9c4a1813c"
+dependencies = [
+ "log",
+ "ring",
+ "rustls-pki-types",
+ "rustls-webpki",
+ "subtle",
+ "zeroize",
+]
+
+[[package]]
+name = "rustls-pemfile"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
+dependencies = [
+ "base64 0.21.5",
+]
+
+[[package]]
+name = "rustls-pki-types"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "868e20fada228fefaf6b652e00cc73623d54f8171e7352c18bb281571f2d92da"
+
+[[package]]
+name = "rustls-webpki"
+version = "0.102.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610"
+dependencies = [
+ "ring",
+ "rustls-pki-types",
+ "untrusted",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
+
+[[package]]
+name = "schannel"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
+dependencies = [
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "schemars"
+version = "0.8.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1847b767a3d62d95cbf3d8a9f0e421cf57a0d8aa4f411d4b16525afb0284d4ed"
+dependencies = [
+ "dyn-clone",
+ "schemars_derive",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "schemars_derive"
+version = "0.8.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af4d7e1b012cb3d9129567661a63755ea4b8a7386d339dc945ae187e403c6743"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "serde_derive_internals",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "secrecy"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bd1c54ea06cfd2f6b63219704de0b9b4f72dcc2b8fdef820be6cd799780e91e"
+dependencies = [
+ "serde",
+ "zeroize",
+]
+
+[[package]]
+name = "security-framework"
+version = "2.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c4437699b6d34972de58652c68b98cb5b53a4199ab126db8e20ec8ded29a721"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.188"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde-value"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3a1a3341211875ef120e117ea7fd5228530ae7e7036a779fdc9117be6b3282c"
+dependencies = [
+ "ordered-float",
+ "serde",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.188"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "serde_derive_internals"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.97"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdf3bf93142acad5821c99197022e170842cdbc1c30482b98750c688c640842a"
+dependencies = [
+ "indexmap 1.9.3",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_spanned"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96426c9936fd7a0124915f9185ea1d20aa9445cc9821142f0a73bc9207a2e186"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "serde_urlencoded"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
+dependencies = [
+ "form_urlencoded",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_yaml"
+version = "0.8.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b"
+dependencies = [
+ "indexmap 1.9.3",
+ "ryu",
+ "serde",
+ "yaml-rust",
+]
+
+[[package]]
+name = "sha2"
+version = "0.10.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8"
+dependencies = [
+ "cfg-if",
+ "cpufeatures",
+ "digest",
+]
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "similar"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32fea41aca09ee824cc9724996433064c89f7777e60762749a4170a14abbfa21"
+
+[[package]]
+name = "slab"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+
+[[package]]
+name = "socket2"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "socket2"
+version = "0.5.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "spin"
+version = "0.9.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
+
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
+[[package]]
+name = "subtle"
+version = "2.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc"
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.37"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7303ef2c05cd654186cb250d29049a24840ca25d2747c25c0381c8d9e2f582e8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "sync_wrapper"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+
+[[package]]
+name = "sysinfo"
+version = "0.29.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd727fc423c2060f6c92d9534cef765c65a6ed3f428a03d7def74a8c4348e666"
+dependencies = [
+ "cfg-if",
+ "core-foundation-sys",
+ "libc",
+ "ntapi",
+ "once_cell",
+ "rayon",
+ "winapi",
+]
+
+[[package]]
+name = "system-configuration"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "tempfile"
+version = "3.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "31c0432476357e58790aaa47a8efb0c5138f137343f3b5f23bd36a27e3b0a6d6"
+dependencies = [
+ "autocfg",
+ "cfg-if",
+ "fastrand",
+ "redox_syscall 0.3.5",
+ "rustix",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "termcolor"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "termtree"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b"
+
+[[package]]
+name = "textwrap"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23d434d3f8967a09480fb04132ebe0a3e088c173e6d0ee7897abbdf4eab0f8b9"
+
+[[package]]
+name = "thiserror"
+version = "1.0.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "thread_local"
+version = "1.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
+[[package]]
+name = "tokio"
+version = "1.28.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f"
+dependencies = [
+ "autocfg",
+ "bytes",
+ "libc",
+ "mio",
+ "num_cpus",
+ "pin-project-lite",
+ "signal-hook-registry",
+ "socket2 0.4.9",
+ "tokio-macros",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "tokio-io-timeout"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf"
+dependencies = [
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-retry"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
+dependencies = [
+ "pin-project",
+ "rand 0.8.5",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-rustls"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f"
+dependencies = [
+ "rustls",
+ "rustls-pki-types",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-stream"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-test"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e89b3cbabd3ae862100094ae433e1def582cf86451b4e9bf83aa7ac1d8a7d719"
+dependencies = [
+ "async-stream",
+ "bytes",
+ "futures-core",
+ "tokio",
+ "tokio-stream",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.6.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "36943ee01a6d67977dd3f84a5a1d2efeb4ada3a1ae771cadfaa535d9d9fc6507"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "log",
+ "pin-project-lite",
+ "slab",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f988a1a1adc2fb21f9c12aa96441da33a1728193ae0b95d2be22dbd17fcb4e5c"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+ "tracing",
+]
+
+[[package]]
+name = "toml"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c17e963a819c331dcacd7ab957d80bc2b9a9c1e71c804826d2f283dd65306542"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.19.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a"
+dependencies = [
+ "indexmap 2.2.6",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
+[[package]]
+name = "tower"
+version = "0.4.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
+dependencies = [
+ "futures-core",
+ "futures-util",
+ "pin-project",
+ "pin-project-lite",
+ "tokio",
+ "tokio-util 0.7.2",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-http"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8"
+dependencies = [
+ "base64 0.13.1",
+ "bitflags 1.3.2",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "http 0.2.9",
+ "http-body 0.4.5",
+ "http-range-header",
+ "pin-project-lite",
+ "tower-layer",
+ "tower-service",
+ "tracing",
+]
+
+[[package]]
+name = "tower-layer"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0"
+
+[[package]]
+name = "tower-service"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+
+[[package]]
+name = "tower-test"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4546773ffeab9e4ea02b8872faa49bb616a80a7da66afc2f32688943f97efa7"
+dependencies = [
+ "futures-util",
+ "pin-project",
+ "tokio",
+ "tokio-test",
+ "tower-layer",
+ "tower-service",
+]
+
+[[package]]
+name = "tracing"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160"
+dependencies = [
+ "cfg-if",
+ "log",
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "treediff"
+version = "3.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "761e8d5ad7ce14bb82b7e61ccc0ca961005a275a060b9644a2431aa11553c2ff"
+dependencies = [
+ "serde_json",
+]
+
+[[package]]
+name = "try-lock"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed"
+
+[[package]]
+name = "typenum"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825"
+
+[[package]]
+name = "unicase"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89"
+dependencies = [
+ "version_check",
+]
+
+[[package]]
+name = "unicode-bidi"
+version = "0.3.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "untrusted"
+version = "0.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
+
+[[package]]
+name = "url"
+version = "2.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "percent-encoding",
+]
+
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
+name = "version_check"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7daec296f25a1bae309c0cd5c29c4b260e510e6d813c286b19eaadf409d40fce"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e397f4664c0e4e428e8313a469aaa58310d302159845980fd23b0f22a847f217"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9afec9963e3d0994cac82455b2b3502b81a7f40f9a0d32181f7528d9f4b43e02"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5961017b3b08ad5f3fe39f1e79877f8ee7c23c5e5fd5eb80de95abc41f1f16b2"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.37",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.88"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d046c5d029ba91a1ed14da14dca44b68bf2f124cfbaf741c54151fdb3e0750b"
+
+[[package]]
+name = "web-sys"
+version = "0.3.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5db499c5f66323272151db0e666cd34f78617522fb0c1604d31a27c50c206a85"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "webpki-roots"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3de34ae270483955a94f4b21bdaaeb83d508bb84a01435f393818edb0012009"
+dependencies = [
+ "rustls-pki-types",
+]
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+
+[[package]]
+name = "windows-core"
+version = "0.51.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64"
+dependencies = [
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.4",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.4",
+ "windows_aarch64_msvc 0.52.4",
+ "windows_i686_gnu 0.52.4",
+ "windows_i686_msvc 0.52.4",
+ "windows_x86_64_gnu 0.52.4",
+ "windows_x86_64_gnullvm 0.52.4",
+ "windows_x86_64_msvc 0.52.4",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8"
+
+[[package]]
+name = "winnow"
+version = "0.5.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "winreg"
+version = "0.50.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
+dependencies = [
+ "cfg-if",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "yaml-rust"
+version = "0.4.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85"
+dependencies = [
+ "linked-hash-map",
+]
+
+[[package]]
+name = "zeroize"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9"
diff --git a/KubeOS-Rust/Cargo.toml b/KubeOS-Rust/Cargo.toml
new file mode 100644
index 0000000000000000000000000000000000000000..9337982324d3fc08cd28c802fec791aba7cb7ccf
--- /dev/null
+++ b/KubeOS-Rust/Cargo.toml
@@ -0,0 +1,12 @@
+[workspace]
+members = ["agent", "cli", "kbimg", "manager", "operator", "proxy"]
+resolver = "2"
+
+[profile.release]
+debug = false
+debug-assertions = false
+lto = true
+opt-level = 's'
+overflow-checks = false
+panic = "unwind"
+rpath = false
diff --git a/KubeOS-Rust/agent/Cargo.toml b/KubeOS-Rust/agent/Cargo.toml
new file mode 100644
index 0000000000000000000000000000000000000000..83e1b7c0b92369b5212d0f557f9447b590757cab
--- /dev/null
+++ b/KubeOS-Rust/agent/Cargo.toml
@@ -0,0 +1,20 @@
+[package]
+description = "KubeOS os-agent"
+edition = "2021"
+license = "MulanPSL-2.0"
+name = "os-agent"
+version = "1.0.6"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[dependencies]
+anyhow = { version = "1.0" }
+env_logger = { version = "0.9" }
+jsonrpc-core = { version = "18.0" }
+jsonrpc-derive = { version = "18.0" }
+jsonrpc-ipc-server = { version = "18.0" }
+lazy_static = { version = "1.4" }
+log = { version = "= 0.4.15" }
+manager = { package = "manager", path = "../manager" }
+nix = { version = "0.26.2" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0" }
diff --git a/KubeOS-Rust/agent/src/function.rs b/KubeOS-Rust/agent/src/function.rs
new file mode 100644
index 0000000000000000000000000000000000000000..9789d95cd081a58030ea1852b935de9c404622bd
--- /dev/null
+++ b/KubeOS-Rust/agent/src/function.rs
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+pub use jsonrpc_core::Result as RpcResult;
+use jsonrpc_core::{Error, ErrorCode};
+pub use jsonrpc_derive::rpc;
+use log::error;
+
+const RPC_OP_ERROR: i64 = -1;
+
+pub struct RpcFunction;
+
+impl RpcFunction {
+    pub fn call<F, T>(f: F) -> RpcResult<T>
+    where
+        F: FnOnce() -> anyhow::Result<T>,
+    {
+        (f)().map_err(|e| {
+            let error_message = format!("{:#}", e);
+            error!("{}", error_message.replace('\n', " ").replace('\r', ""));
+            Error { code: ErrorCode::ServerError(RPC_OP_ERROR), message: format!("{:?}", e), data: None }
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_rpcfunction_call() {
+        // Define a mock function that returns a result
+        fn mock_ok_function() -> anyhow::Result<u32> {
+            Ok(42)
+        }
+        let result: RpcResult<u32> = RpcFunction::call(mock_ok_function);
+        assert!(result.is_ok());
+        assert_eq!(result.unwrap(), 42);
+
+        fn mock_err_function() -> anyhow::Result<u32> {
+            Err(anyhow::anyhow!("error"))
+        }
+        let result: RpcResult<u32> = RpcFunction::call(mock_err_function);
+        assert!(result.is_err());
+        assert_eq!(result.unwrap_err().code, ErrorCode::ServerError(RPC_OP_ERROR));
+    }
+}
diff --git a/KubeOS-Rust/agent/src/main.rs b/KubeOS-Rust/agent/src/main.rs
new file mode 100644
index 0000000000000000000000000000000000000000..cd95ef072e80731ff90f7a9cf2316919853b083d
--- /dev/null
+++ b/KubeOS-Rust/agent/src/main.rs
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{
+    fs::{self, DirBuilder, Permissions},
+    os::unix::fs::{DirBuilderExt, PermissionsExt},
+    path::Path,
+};
+
+use env_logger::{Builder, Env, Target};
+use jsonrpc_core::{IoHandler, IoHandlerExtension};
+use jsonrpc_ipc_server::ServerBuilder;
+
+mod function;
+mod rpc;
+
+use log::info;
+use rpc::{Agent, AgentImpl};
+
+const SOCK_PATH: &str = "/run/os-agent/os-agent.sock";
+const CARGO_PKG_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION");
+
+fn start_and_run(sock_path: &str) {
+    let socket_path = Path::new(sock_path);
+
+    // Create directory for socket if it doesn't exist
+    if let Some(dir_path) = socket_path.parent() {
+        if !dir_path.exists() {
+            DirBuilder::new().mode(0o750).create(dir_path).expect("Couldn't create directory for socket");
+        }
+    }
+
+    // Add RPC methods to IoHandler
+    let mut io = IoHandler::new();
+    AgentImpl::default().to_delegate().augment(&mut io);
+
+    // Build and start server
+    let builder = ServerBuilder::new(io);
+    let server = builder.start(sock_path).expect("Couldn't open socket");
+
+    let gid = nix::unistd::getgid();
+    nix::unistd::chown(socket_path, Some(nix::unistd::ROOT), Some(gid)).expect("Couldn't set socket group");
+
+    // Set socket permissions to 0640
+    let socket_permissions = Permissions::from_mode(0o640);
+    fs::set_permissions(socket_path, socket_permissions).expect("Couldn't set socket permissions");
+
+    info!("os-agent started, waiting for requests...");
+    server.wait();
+}
+
+fn main() {
+    Builder::from_env(Env::default().default_filter_or("info")).target(Target::Stdout).init();
+
+    info!("os-agent version is: {}", CARGO_PKG_VERSION.unwrap_or("NOT FOUND"));
+    start_and_run(SOCK_PATH);
+}
diff --git a/KubeOS-Rust/agent/src/rpc/agent.rs b/KubeOS-Rust/agent/src/rpc/agent.rs
new file mode 100644
index 0000000000000000000000000000000000000000..2496bfb4b807eba584491830d5f0c11b015a6afe
--- /dev/null
+++ b/KubeOS-Rust/agent/src/rpc/agent.rs
@@ -0,0 +1,30 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use manager::api::{ConfigureRequest, Response, UpgradeRequest};
+
+use super::function::{rpc, RpcResult};
+
+#[rpc(server)]
+pub trait Agent {
+    #[rpc(name = "prepare_upgrade")]
+    fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult<Response>;
+
+    #[rpc(name = "upgrade")]
+    fn upgrade(&self) -> RpcResult<Response>;
+
+    #[rpc(name = "configure")]
+    fn configure(&self, req: ConfigureRequest) -> RpcResult<Response>;
+
+    #[rpc(name = "rollback")]
+    fn rollback(&self) -> RpcResult<Response>;
+}
diff --git a/KubeOS-Rust/agent/src/rpc/agent_impl.rs b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
new file mode 100644
index 0000000000000000000000000000000000000000..ab826413559cab6ed3240633c78a8325b072ab38
--- /dev/null
+++ b/KubeOS-Rust/agent/src/rpc/agent_impl.rs
@@ -0,0 +1,217 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{sync::Mutex, thread, time::Duration};
+
+use anyhow::{bail, Result};
+use log::{debug, info};
+use manager::{
+    api::{AgentStatus, ConfigureRequest, ImageType, Response, UpgradeRequest},
+    sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler, CONFIG_TEMPLATE, DEFAULT_GRUBENV_PATH},
+    utils::{get_partition_info, switch_boot_menuentry, RealCommandExecutor},
+};
+use nix::{sys::reboot::RebootMode, unistd::sync};
+
+use super::{
+    agent::Agent,
+    function::{RpcFunction, RpcResult},
+};
+
+pub struct AgentImpl {
+    mutex: Mutex<()>,
+    disable_reboot: bool,
+}
+
+impl Agent for AgentImpl {
+    fn prepare_upgrade(&self, req: UpgradeRequest) -> RpcResult<Response> {
+        RpcFunction::call(|| self.prepare_upgrade_impl(req))
+    }
+
+    fn upgrade(&self) -> RpcResult<Response> {
+        RpcFunction::call(|| self.upgrade_impl())
+    }
+
+    fn configure(&self, req: ConfigureRequest) -> RpcResult<Response> {
+        RpcFunction::call(|| self.configure_impl(req))
+    }
+
+    fn rollback(&self) -> RpcResult<Response> {
+        RpcFunction::call(|| self.rollback_impl())
+    }
+}
+
+impl Default for AgentImpl {
+    fn default() -> Self {
+        Self { mutex: Mutex::new(()), disable_reboot: false }
+    }
+}
+
+impl AgentImpl {
+    fn prepare_upgrade_impl(&self, req: UpgradeRequest) -> Result<Response> {
+        let lock = self.mutex.try_lock();
+        if lock.is_err() {
+            bail!("os-agent is processing another request");
+        }
+        debug!("Received an 'prepare upgrade' request: {:?}", req);
+        info!("Start preparing for upgrading to version: {}", req.version);
+
+        let handler: Box<ImageType<RealCommandExecutor>> = match req.image_type.as_str() {
+            "containerd" => Box::new(ImageType::Containerd(CtrImageHandler::default())),
+            "docker" => Box::new(ImageType::Docker(DockerImageHandler::default())),
+            "disk" => Box::new(ImageType::Disk(DiskImageHandler::default())),
+            _ => bail!("Invalid image type \"{}\"", req.image_type),
+        };
+
+        let image_manager = handler.download_image(&req)?;
+        info!("Ready to install image: {:?}", image_manager.paths.image_path.display());
+        image_manager.install()?;
+
+        Ok(Response { status: AgentStatus::UpgradeReady })
+    }
+
+    fn upgrade_impl(&self) -> Result<Response> {
+        let lock = self.mutex.try_lock();
+        if lock.is_err() {
+            bail!("os-agent is processing another request");
+        }
+        info!("Start to upgrade");
+        let command_executor = RealCommandExecutor {};
+        let (_, next_partition_info) = get_partition_info(&command_executor)?;
+
+        // based on boot mode use different command to switch boot partition
+        let device = next_partition_info.device.as_str();
+        let menuentry = next_partition_info.menuentry.as_str();
+        switch_boot_menuentry(&command_executor, DEFAULT_GRUBENV_PATH, menuentry)?;
+        info!("Switch to boot partition: {}, device: {}", menuentry, device);
+        self.reboot()?;
+        Ok(Response { status: AgentStatus::Upgraded })
+    }
+
+    fn configure_impl(&self, mut req: ConfigureRequest) -> Result<Response> {
+        let lock = self.mutex.try_lock();
+        if lock.is_err() {
+            bail!("os-agent is processing another request");
+        }
+        debug!("Received a 'configure' request: {:?}", req);
+        info!("Start to configure");
+        let config_map = &*CONFIG_TEMPLATE;
+        for config in req.configs.iter_mut() {
+            let config_type = &config.model;
+            if let Some(configuration) = config_map.get(config_type) {
+                debug!("Found configuration type: \"{}\"", config_type);
+                configuration.set_config(config)?;
+            } else {
+                bail!("Unknown configuration type: \"{}\"", config_type);
+            }
+        }
+        Ok(Response { status: AgentStatus::Configured })
+    }
+
+    fn rollback_impl(&self) -> Result<Response> {
+        let lock = self.mutex.try_lock();
+        if lock.is_err() {
+            bail!("os-agent is processing another request");
+        }
+        info!("Start to rollback");
+        let command_executor = RealCommandExecutor {};
+        let (_, next_partition_info) = get_partition_info(&command_executor)?;
+        switch_boot_menuentry(
+            &command_executor,
+            manager::sys_mgmt::DEFAULT_GRUBENV_PATH,
+            &next_partition_info.menuentry,
+        )?;
+        info!("Switch to boot partition: {}, device: {}", next_partition_info.menuentry, next_partition_info.device);
+        self.reboot()?;
+        Ok(Response { status: AgentStatus::Rollbacked })
+    }
+
+    fn reboot(&self) -> Result<()> {
+        info!("Wait to reboot");
+        thread::sleep(Duration::from_secs(1));
+        sync();
+        if self.disable_reboot {
+            return Ok(());
+        }
+        nix::sys::reboot::reboot(RebootMode::RB_AUTOBOOT)?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashMap;
+
+    use manager::api::{CertsInfo, Sysconfig};
+
+    use super::*;
+
+    #[test]
+    fn test_reboot() {
+        let mut agent = AgentImpl::default();
+        agent.disable_reboot = true;
+        let res = agent.reboot();
+        assert!(res.is_ok());
+    }
+
+    #[test]
+    fn test_configure() {
+        let agent = AgentImpl::default();
+        let req = ConfigureRequest {
+            configs: vec![Sysconfig {
+                model: "kernel.sysctl".to_string(),
+                config_path: "".to_string(),
+                contents: HashMap::new(),
+            }],
+        };
+        let res = agent.configure(req).unwrap();
+        assert_eq!(res, Response { status: AgentStatus::Configured });
+
+        let req = ConfigureRequest {
+            configs: vec![Sysconfig {
+                model: "invalid".to_string(),
+                config_path: "".to_string(),
+                contents: HashMap::new(),
+            }],
+        };
+        let res = agent.configure(req);
+        assert!(res.is_err());
+
+        // test lock
+        let _lock = agent.mutex.lock().unwrap();
+        let req = ConfigureRequest {
+            configs: vec![Sysconfig {
+                model: "kernel.sysctl".to_string(),
+                config_path: "".to_string(),
+                contents: HashMap::new(),
+            }],
+        };
+        let res = agent.configure(req);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_prepare_upgrade() {
+        let agent = AgentImpl::default();
+        let req = UpgradeRequest {
+            version: "v2".into(),
+            check_sum: "xxx".into(),
+            image_type: "xxx".into(),
+            container_image: "xxx".into(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        let res = agent.prepare_upgrade(req);
+        assert!(res.is_err());
+    }
+}
diff --git a/KubeOS-Rust/agent/src/rpc/mod.rs b/KubeOS-Rust/agent/src/rpc/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..976356be57ebc429e0aa6ca5d5cf350d5a16ad6d
--- /dev/null
+++ b/KubeOS-Rust/agent/src/rpc/mod.rs
@@ -0,0 +1,19 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use super::function;
+
+mod agent;
+mod agent_impl;
+
+pub use agent::*;
+pub use agent_impl::*;
diff --git a/KubeOS-Rust/cli/Cargo.toml b/KubeOS-Rust/cli/Cargo.toml
new file mode 100644
index 0000000000000000000000000000000000000000..78d5fd5185ab242eac57be1f7843aed5ecf746ab
--- /dev/null
+++ b/KubeOS-Rust/cli/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+description = "KubeOS os-agent client"
+edition = "2021"
+license = "MulanPSL-2.0"
+name = "cli"
+version = "1.0.6"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[dependencies]
+anyhow = { version = "1.0" }
+jsonrpc = { version = "0.13", features = ["simple_uds"] }
+kubeos-manager = { package = "manager", path = "../manager" }
+log = { version = "0.4" }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0" }
diff --git a/KubeOS-Rust/cli/src/client.rs b/KubeOS-Rust/cli/src/client.rs
new file mode 100644
index 0000000000000000000000000000000000000000..37518bdc9f4ccd4339e5405f3638aa43e0f5e29b
--- /dev/null
+++ b/KubeOS-Rust/cli/src/client.rs
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::path::Path;
+
+use jsonrpc::{
+    simple_uds::UdsTransport, Client as JsonRPCClient, Request as JsonRPCRequest, Response as JsonRPCResponse,
+};
+use serde_json::value::RawValue;
+
+pub struct Client {
+    json_rpc_client: JsonRPCClient,
+}
+
+pub struct Request<'a>(JsonRPCRequest<'a>);
+
+impl<'a> Request<'a> {}
+
+impl Client {
+    pub fn new<P: AsRef<Path>>(socket_path: P) -> Self {
+        Client { json_rpc_client: JsonRPCClient::with_transport(UdsTransport::new(socket_path)) }
+    }
+
+    pub fn build_request<'a>(&self, command: &'a str, params: &'a [Box<RawValue>]) -> Request<'a> {
+        let json_rpc_request = self.json_rpc_client.build_request(command, params);
+        let request = Request(json_rpc_request);
+        request
+    }
+
+    pub fn send_request(&self, request: Request) -> Result<JsonRPCResponse, jsonrpc::Error> {
+        self.json_rpc_client.send_request(request.0)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    #[test]
+    fn test_client() {
+        let socket_path = "/tmp/KubeOS-test.sock";
+        let cli = Client::new(socket_path);
+        let command = "example_command";
+        let params = vec![];
+        let request = cli.send_request(cli.build_request(command, &params));
+        assert!(request.is_err());
+    }
+}
diff --git a/KubeOS-Rust/cli/src/lib.rs b/KubeOS-Rust/cli/src/lib.rs
new file mode 100644
index 0000000000000000000000000000000000000000..cd66d72f0f30fb7cd061eac998fabf4424c26c06
--- /dev/null
+++ b/KubeOS-Rust/cli/src/lib.rs
@@ -0,0 +1,14 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+pub mod client;
+pub mod method;
diff --git a/KubeOS-Rust/cli/src/method/callable_method.rs b/KubeOS-Rust/cli/src/method/callable_method.rs
new file mode 100644
index 0000000000000000000000000000000000000000..a174b5b8fac8db9d4388ff960fe9d9ee6ee336c1
--- /dev/null
+++ b/KubeOS-Rust/cli/src/method/callable_method.rs
@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use serde_json::value::RawValue;
+
+use super::request::{parse_error, request};
+use crate::client::Client;
+
+pub trait RpcMethod {
+    type Response: serde::de::DeserializeOwned;
+    fn command_name(&self) -> &'static str;
+    fn command_params(&self) -> Vec<Box<RawValue>>;
+    fn call(&self, client: &Client) -> Result<Self::Response, anyhow::Error> {
+        let response = request(client, self.command_name(), self.command_params())?;
+        response.result().map_err(parse_error)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::client;
+
+    #[derive(Default)]
+    struct DummyMethod;
+
+    impl RpcMethod for DummyMethod {
+        type Response = String;
+
+        fn command_name(&self) -> &'static str {
+            "dummy_command"
+        }
+
+        fn command_params(&self) -> Vec<Box<RawValue>> {
+            vec![]
+        }
+    }
+
+    #[test]
+    fn test_call() {
+        let client = client::Client::new("/tmp/KubeOS-test.sock");
+        let result = DummyMethod::default().call(&client);
+        assert!(result.is_err());
+    }
+}
diff --git a/KubeOS-Rust/cli/src/method/configure.rs b/KubeOS-Rust/cli/src/method/configure.rs
new file mode 100644
index 0000000000000000000000000000000000000000..cca752d0a39b7c5073d9de57fdee5fbd3be04e75
--- /dev/null
+++ b/KubeOS-Rust/cli/src/method/configure.rs
@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use kubeos_manager::api;
+use serde_json::value::{to_raw_value, RawValue};
+
+use crate::method::callable_method::RpcMethod;
+
+pub struct ConfigureMethod {
+    req: api::ConfigureRequest,
+}
+
+impl ConfigureMethod {
+    pub fn new(req: api::ConfigureRequest) -> Self {
+        ConfigureMethod { req }
+    }
+
+    pub fn set_configure_request(&mut self, req: api::ConfigureRequest) -> &Self {
+        self.req = req;
+        self
+    }
+}
+
+impl RpcMethod for ConfigureMethod {
+    type Response = api::Response;
+    fn command_name(&self) -> &'static str {
+        "configure"
+    }
+    fn command_params(&self) -> Vec<Box<RawValue>> {
+        vec![to_raw_value(&self.req).unwrap()]
+    }
+}
+#[cfg(test)]
+mod tests {
+    use kubeos_manager::api::{ConfigureRequest, Sysconfig};
+
+    use super::*;
+
+    #[test]
+    fn test_configure_method() {
+        let req = ConfigureRequest { configs: vec![] };
+        let mut method = ConfigureMethod::new(req);
+
+        // Test set_configure_request method
+        let new_req = ConfigureRequest {
+            configs: vec![Sysconfig {
+                model: "model".to_string(),
+                config_path: "config_path".to_string(),
+                contents: Default::default(),
+            }],
+        };
+        method.set_configure_request(new_req);
+
+        // Test command_name method
+        assert_eq!(method.command_name(), "configure");
+
+        // Test command_params method
+        let expected_params =
+            "RawValue({\"configs\":[{\"model\":\"model\",\"config_path\":\"config_path\",\"contents\":{}}]})";
+        let actual_params = format!("{:?}", method.command_params()[0]);
+        assert_eq!(actual_params, expected_params);
+    }
+}
diff --git a/KubeOS-Rust/cli/src/method/mod.rs b/KubeOS-Rust/cli/src/method/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..e1f38bcd24d860ab68fba0ef7a6b09846aa4d8f7
--- /dev/null
+++ b/KubeOS-Rust/cli/src/method/mod.rs
@@ -0,0 +1,18 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+pub mod callable_method;
+pub mod configure;
+pub mod prepare_upgrade;
+pub mod request;
+pub mod rollback;
+pub mod upgrade;
diff --git a/KubeOS-Rust/cli/src/method/prepare_upgrade.rs b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs
new file mode 100644
index 0000000000000000000000000000000000000000..f2034f6b7cd2d3d9bf2779e749bac6f5e7e5cbfd
--- /dev/null
+++ b/KubeOS-Rust/cli/src/method/prepare_upgrade.rs
@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use kubeos_manager::api;
+use serde_json::value::{to_raw_value, RawValue};
+
+use crate::method::callable_method::RpcMethod;
+
+pub struct PrepareUpgradeMethod {
+    req: api::UpgradeRequest,
+}
+
+impl PrepareUpgradeMethod {
+    pub fn new(req: api::UpgradeRequest) -> Self {
+        PrepareUpgradeMethod { req }
+    }
+
+    pub fn set_prepare_upgrade_request(&mut self, req: api::UpgradeRequest) -> &Self {
+        self.req = req;
+        self
+    }
+}
+
+impl RpcMethod for PrepareUpgradeMethod {
+    type Response = api::Response;
+    fn command_name(&self) -> &'static str {
+        "prepare_upgrade"
+    }
+    fn command_params(&self) -> Vec<Box<RawValue>> {
+        vec![to_raw_value(&self.req).unwrap()]
+    }
+}
+#[cfg(test)]
+mod tests {
+    use kubeos_manager::api::{CertsInfo, UpgradeRequest};
+
+    use super::*;
+
+    #[test]
+    fn test_prepare_upgrade_method() {
+        let req = UpgradeRequest {
+            version: "v1".into(),
+            check_sum: "".into(),
+            image_type: "".into(),
+            container_image: "".into(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        let mut method = PrepareUpgradeMethod::new(req);
+        let new_req = UpgradeRequest {
+            version: "v2".into(),
+            check_sum: "xxx".into(),
+            image_type: "xxx".into(),
+            container_image: "xxx".into(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        method.set_prepare_upgrade_request(new_req);
+        assert_eq!(method.command_name(), "prepare_upgrade");
+
+        let expected_params = "RawValue({\"version\":\"v2\",\"check_sum\":\"xxx\",\"image_type\":\"xxx\",\"container_image\":\"xxx\",\"image_url\":\"\",\"flag_safe\":false,\"mtls\":false,\"certs\":{\"ca_cert\":\"\",\"client_cert\":\"\",\"client_key\":\"\"}})";
+        let actual_params = format!("{:?}", method.command_params()[0]);
+        assert_eq!(actual_params, expected_params);
+    }
+}
diff --git a/KubeOS-Rust/cli/src/method/request.rs b/KubeOS-Rust/cli/src/method/request.rs
new file mode 100644
index 0000000000000000000000000000000000000000..581aa6392baa98b7d2e5689800f4087abab5f617
--- /dev/null
+++ b/KubeOS-Rust/cli/src/method/request.rs
@@ -0,0 +1,88 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use anyhow::anyhow;
+use jsonrpc::{Error, Response};
+use log::debug;
+use serde_json::value::RawValue;
+
+use crate::client::Client;
+
+pub fn request(client: &Client, command: &str, params: Vec<Box<RawValue>>) -> Result<Response, anyhow::Error> {
+    let request = client.build_request(command, &params);
+    let response = client.send_request(request).map_err(parse_error);
+    debug!("{:#?}", response);
+    response
+}
+
+pub fn parse_error(error: Error) -> anyhow::Error {
+    match error {
+        Error::Transport(e) => {
+            anyhow!(
+                "Cannot connect to KubeOS os-agent unix socket, {}",
+                e.source().map(|e| e.to_string()).unwrap_or_else(|| "Connection timeout".to_string())
+            )
+        },
+        Error::Json(e) => {
+            debug!("Json parse error: {:?}", e);
+            anyhow!("Failed to parse response")
+        },
+        Error::Rpc(ref e) => {
+            if e.message == "Method not found" {
+                anyhow!("Method is unimplemented")
+            } else {
+                anyhow!("{}", e.message)
+            }
+        },
+        _ => {
+            debug!("{:?}", error);
+            anyhow!("Response is invalid")
+        },
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use jsonrpc::error::RpcError;
+    use serde::de::Error as DeError;
+
+    use super::*;
+
+    #[test]
+    fn test_parse_error() {
+        // Test Error::Transport
+        let transport_error =
+            Error::Transport(Box::new(std::io::Error::new(std::io::ErrorKind::Other, "Connection timeout")));
+        let result = parse_error(transport_error);
+        assert_eq!(result.to_string(), "Cannot connect to KubeOS os-agent unix socket, Connection timeout");
+
+        // Test Error::Json
+        let json_error = Error::Json(serde_json::Error::custom("Failed to parse response"));
+        let result = parse_error(json_error);
+        assert_eq!(result.to_string(), "Failed to parse response");
+
+        // Test Error::Rpc with "Method not found" message
+        let rpc_error = Error::Rpc(RpcError { code: -32601, message: "Method not found".to_string(), data: None });
+        let result = parse_error(rpc_error);
+        assert_eq!(result.to_string(), "Method is unimplemented");
+
+        // Test Error::Rpc with other message
+        let rpc_error = Error::Rpc(RpcError { code: -32603, message: "Internal server error".to_string(), data: None });
+        let result = parse_error(rpc_error);
+        assert_eq!(result.to_string(), "Internal server error");
+
+        // Test other Error variant
+        let other_error = Error::VersionMismatch;
+        let result = parse_error(other_error);
+        assert_eq!(result.to_string(), "Response is invalid");
+    }
+}
diff --git a/KubeOS-Rust/cli/src/method/rollback.rs b/KubeOS-Rust/cli/src/method/rollback.rs
new file mode 100644
index 0000000000000000000000000000000000000000..7945f4b16c60b14ecbd4d478e6ac3dab4196980b
--- /dev/null
+++ b/KubeOS-Rust/cli/src/method/rollback.rs
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use kubeos_manager::api;
+use serde_json::value::RawValue;
+
+use crate::method::callable_method::RpcMethod;
+
+#[derive(Default)]
+pub struct RollbackMethod {}
+
+impl RpcMethod for RollbackMethod {
+    type Response = api::Response;
+    fn command_name(&self) -> &'static str {
+        "rollback"
+    }
+    fn command_params(&self) -> Vec<Box<RawValue>> {
+        vec![]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_rollback_method() {
+        let method = RollbackMethod::default();
+        assert_eq!(method.command_name(), "rollback");
+        let expected_params = "[]";
+        let actual_params = format!("{:?}", method.command_params());
+        assert_eq!(actual_params, expected_params);
+    }
+}
diff --git a/KubeOS-Rust/cli/src/method/upgrade.rs b/KubeOS-Rust/cli/src/method/upgrade.rs
new file mode 100644
index 0000000000000000000000000000000000000000..f2f94cd5ead367a64f0a1461d11d47b2a1e1c410
--- /dev/null
+++ b/KubeOS-Rust/cli/src/method/upgrade.rs
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use kubeos_manager::api;
+use serde_json::value::RawValue;
+
+use crate::method::callable_method::RpcMethod;
+
+#[derive(Default)]
+pub struct UpgradeMethod {}
+
+impl RpcMethod for UpgradeMethod {
+    type Response = api::Response;
+    fn command_name(&self) -> &'static str {
+        "upgrade"
+    }
+    fn command_params(&self) -> Vec<Box<RawValue>> {
+        vec![]
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    #[test]
+    fn test_upgrade_method() {
+        let method = UpgradeMethod::default();
+        assert_eq!(method.command_name(), "upgrade");
+        let expected_params = "[]";
+        let actual_params = format!("{:?}", method.command_params());
+        assert_eq!(actual_params, expected_params);
+    }
+}
diff --git a/KubeOS-Rust/kbimg/Cargo.toml b/KubeOS-Rust/kbimg/Cargo.toml
new file mode 100644
index 0000000000000000000000000000000000000000..cec12b8d938080685fdd07147f5abaae9bc7fcdb
--- /dev/null
+++ b/KubeOS-Rust/kbimg/Cargo.toml
@@ -0,0 +1,16 @@
+[package]
+description = "KubeOS kbimg"
+edition = "2021"
+license = "MulanPSL-2.0"
+name = "kbimg"
+version = "1.0.6"
+
+[dependencies]
+anyhow = { version = "1.0" }
+clap = { version = "=3.2.23", features = ["derive"] }
+env_logger = { version = "0.9" }
+log = { version = "= 0.4.15" }
+regex = { version = "1.7.3" }
+serde = { version = "1.0", features = ["derive"] }
+sysinfo = { version = "=0.29.11" }
+toml = { version = "=0.7.6" }
diff --git a/KubeOS-Rust/kbimg/kbimg.toml b/KubeOS-Rust/kbimg/kbimg.toml
new file mode 100644
index 0000000000000000000000000000000000000000..aaa9b48ffab3cced82716d448b62c8d6b9f2cad6
--- /dev/null
+++ b/KubeOS-Rust/kbimg/kbimg.toml
@@ -0,0 +1,87 @@
+[from_repo]
+agent_path = "/root/KubeOS/bin/os-agent"
+image_type = "vm-repo"
+legacy_bios = false
+repo_path = "/etc/yum.repos.d/openEuler.repo"
+root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0"
+version = "v1"
+rpmlist = [
+    "NetworkManager",
+    "conntrack-tools",
+    "containernetworking-plugins",
+    "coreutils",
+    "dhcp",
+    "docker",
+    "dosfstools",
+    "dracut",
+    "ebtables",
+    "ethtool",
+    "gawk",
+    "hwinfo",
+    "kernel",
+    "kubernetes-kubeadm",
+    "kubernetes-kubelet",
+    "lvm2",
+    "net-tools",
+    "openssh-server",
+    "passwd",
+    "parted",
+    "rsyslog",
+    "socat",
+    "sudo",
+    "vi",
+]
+
+# [from_dockerimg]
+# docker_img = ""
+# image_type = "vm-docker"
+
+# [admin_container]
+# dockerfile = ""
+# docker_img = ""
+
+# [pxe_config]
+# rootfs_name = "kubeos.tar"
+# disk = "/dev/sda"
+# server_ip = "192.168.1.50"
+# local_ip = "192.168.1.100"
+# route_ip = "192.168.1.1"
+# netmask = "255.255.255.0"
+# net_name = "eth0"
+
+# [[users]]
+# groups = ["admin"]
+# name = "foo"
+# passwd = "foo"
+# sudo = "ALL=(ALL) ALL"
+
+# [[users]]
+# groups = ["example"]
+# name = "bar"
+# passwd = "bar"
+
+# [[copy_files]]
+# dst = "/root/ztest2"
+# src = "/root/KubeOS/ztest2"
+
+# [[copy_files]]
+# dst = "/root/ztest1"
+# src = "/root/KubeOS/ztest1/*"
+
+# [grub]
+# passwd = "foo"
+
+# [systemd_service]
+# name = ["kubelet", "containerd"]
+
+# [chroot_script]
+# path = "../../ztest/myscript.sh"
+
+# [disk_partition]
+# first = 100
+# second = 2500
+# third = 2300
+# img_size = 30
+
+# [persist_mkdir]
+# name = ["lv_data"]
diff --git a/KubeOS-Rust/kbimg/src/admin_container.rs b/KubeOS-Rust/kbimg/src/admin_container.rs
new file mode 100644
index 0000000000000000000000000000000000000000..0319673999d7fcb6831eb960c155e16fd4fc8f69
--- /dev/null
+++ b/KubeOS-Rust/kbimg/src/admin_container.rs
@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{
+    fs::{create_dir_all, File},
+    path::PathBuf,
+};
+
+use anyhow::bail;
+
+use crate::{
+    commands::AdminContainerInfo,
+    scripts_gen::*,
+    utils::{self, set_permissions},
+    values::*,
+    Config, CreateImage,
+};
+
+impl CreateImage for AdminContainerInfo {
+    fn prepare(&self, _: &mut Config) -> anyhow::Result<()> {
+        let dockerfile = &self.dockerfile;
+        let image_name = &self.docker_img;
+        verify_admin_input(&dockerfile, &image_name)?;
+        check_dockerfile_valid(&dockerfile)?;
+        Ok(())
+    }
+
+    fn generate_scripts(&self, _: &Config) -> anyhow::Result<PathBuf> {
+        // admin-container
+        match create_dir_all(ADMIN_CONTAINER_DIR) {
+            Ok(_) => {
+                // Dockerfile
+                let dockerfile_path = format!("{}/{}", ADMIN_CONTAINER_DIR, ADMIN_DOCKERFILE);
+                let mut dockerfile = File::create(&dockerfile_path)?;
+                gen_admin_dockerfile(&mut dockerfile)?;
+                set_permissions(&dockerfile_path, CONFIG_PERMISSION)?;
+                // set-ssh-pub-key.service
+                let set_ssh_pub_key_service_path = format!("{}/{}", ADMIN_CONTAINER_DIR, ADMIN_SET_SSH_PUB_KEY_SERVICE);
+                let mut set_ssh_pub_key_service = File::create(&set_ssh_pub_key_service_path)?;
+                gen_set_ssh_pub_key_service(&mut set_ssh_pub_key_service)?;
+                set_permissions(&set_ssh_pub_key_service_path, CONFIG_PERMISSION)?;
+                // set-ssh-pub-key.sh
+                let set_ssh_pub_key_path = format!("{}/{}", ADMIN_CONTAINER_DIR, ADMIN_SET_SSH_PUB_KEY_SH);
+                let mut set_ssh_pub_key = File::create(&set_ssh_pub_key_path)?;
+                gen_set_ssh_pub_key(&mut set_ssh_pub_key)?;
+                set_permissions(&set_ssh_pub_key_path, EXEC_PERMISSION)?;
+            },
+            Err(e) => {
+                bail!(e);
+            },
+        }
+        // kbimg.sh
+        let kbimg_path = format!("{}/{}", SCRIPTS_DIR, KBIMG_SH);
+        let mut kbimg = File::create(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH))?;
+        gen_admin_vars(&mut kbimg, &self.docker_img, &self.dockerfile)?;
+        gen_create_admin_img(&mut kbimg)?;
+        set_permissions(&kbimg_path, EXEC_PERMISSION)?;
+
+        Ok(PathBuf::from(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH)))
+    }
+}
+
+fn verify_admin_input(dockerfile: &PathBuf, image_name: &str) -> anyhow::Result<()> {
+    if !utils::is_valid_param(dockerfile.to_str().unwrap()) {
+        bail!("params {} is invalid, please check input", dockerfile.to_str().unwrap());
+    }
+    if !utils::is_valid_param(image_name) {
+        bail!("params {} is invalid, please check input", image_name);
+    }
+    Ok(())
+}
+
+fn check_dockerfile_valid(dockerfile: &PathBuf) -> anyhow::Result<()> {
+    utils::is_file_valid("admin-container Dockerfile", dockerfile)
+}
diff --git a/KubeOS-Rust/kbimg/src/commands.rs b/KubeOS-Rust/kbimg/src/commands.rs
new file mode 100644
index 0000000000000000000000000000000000000000..f334f0487d242f0e9f366f62bec9bbbc5f3e8886
--- /dev/null
+++ b/KubeOS-Rust/kbimg/src/commands.rs
@@ -0,0 +1,170 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::path::PathBuf;
+
+use clap::{Args, Parser, Subcommand};
+use serde::Deserialize;
+
+#[derive(Parser)]
+#[clap(name = "kbimg")]
+#[clap(author, version, about)]
+#[clap(long_about = "A tool for creating KubeOS images.")]
+pub struct Cli {
+    /// Path to the detailed configuration toml file
+    #[clap(short, long, value_parser)]
+    pub config: Option<PathBuf>,
+    /// Enable debug mode, keep the scripts after execution
+    #[clap(short, long, action)]
+    pub debug: bool,
+    #[clap(subcommand)]
+    pub commands: Option<Commands>,
+}
+
+#[derive(Subcommand, Debug, Deserialize)]
+pub enum Commands {
+    /// Create a new container image for upgrading KubeOS
+    #[clap(name = "upgrade")]
+    UpgradeImage(RepoInfo),
+    /// Create a new KubeOS vm image from repo
+    #[clap(name = "vm-repo")]
+    VMRepo(RepoInfo),
+    /// Create a new KubeOS vm image from docker image
+    #[clap(name = "vm-docker")]
+    VMDocker(DockerInfo),
+    /// Create a new KubeOS pxe image from repo
+    #[clap(name = "pxe-repo")]
+    PxeRepo(RepoInfo),
+    /// Create a new KubeOS pxe image from docker image
+    #[clap(name = "pxe-docker")]
+    PxeDocker(DockerInfo),
+    /// Create a KubeOS admin-container image
+    #[clap(name = "admin-container")]
+    AdminContainer(AdminContainerInfo),
+}
+
+#[derive(Args, Debug, Deserialize, Clone)]
+pub struct RepoInfo {
+    /// Required: KubeOS version
+    #[clap(short, long, value_parser)]
+    pub version: String,
+    /// Required: Repo path for installing packages
+    #[clap(short = 'p', long, value_parser)]
+    pub repo_path: PathBuf,
+    /// Required: Path to the agent binary
+    #[clap(short = 'b', long, value_parser)]
+    pub agent_path: PathBuf,
+    /// Required: Encrypted password for root user
+    #[clap(short = 'e', long, value_parser)]
+    pub root_passwd: String,
+    /// Required for upgrade
+    #[clap(short = 'd', long, value_parser)]
+    pub docker_img: Option<String>,
+    /// Required: RPM packages
+    #[clap(short = 'r', long, value_parser)]
+    pub rpmlist: Vec<String>,
+    /// Optional: boot mode, default is uefi, enable this flag for legacy bios
+    #[clap(short, long, value_parser)]
+    pub legacy_bios: bool,
+    #[clap(skip)]
+    pub image_type: String,
+    #[clap(skip)]
+    pub arch: Option<String>,
+}
+
+#[derive(Args, Debug, Deserialize, Clone)]
+pub struct DockerInfo {
+    /// Required: Name of the container image
+    #[clap(short, long, value_parser)]
+    pub docker_img: String,
+    #[clap(skip)]
+    pub image_type: String,
+}
+
+#[derive(Args, Debug, Deserialize, Clone)]
+pub struct AdminContainerInfo {
+    /// Required: Name of the container image
+    #[clap(short, long, value_parser)]
+    pub docker_img: String,
+    /// Required: Path to the Dockerfile
+    #[clap(short, long, value_parser)]
+    pub dockerfile: PathBuf,
+}
+
+#[derive(Debug, Deserialize, Default, Clone)]
+pub struct Config {
+    pub from_repo: Option<RepoInfo>,
+    pub from_dockerimg: Option<DockerInfo>,
+    pub admin_container: Option<AdminContainerInfo>,
+    pub users: Option<Vec<User>>,
+    pub copy_files: Option<Vec<CopyFile>>,
+    pub grub: Option<Grub>,
+    pub systemd_service: Option<SystemdService>,
+    pub chroot_script: Option<ChrootScript>,
+    pub disk_partition: Option<DiskPartition>,
+    pub persist_mkdir: Option<PersistMkdir>,
+    pub pxe_config: Option<PxeConfig>,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct User {
+    pub name: String,
+    pub passwd: String,
+    pub groups: Option<Vec<String>>,
+    pub sudo: Option<String>,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct CopyFile {
+    pub src: String,
+    pub dst: String,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct Grub {
+    pub passwd: Option<String>,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct SystemdService {
+    pub name: Vec<String>,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct ChrootScript {
+    pub path: String,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct DiskPartition {
+    pub first: u32,
+    pub second: u32,
+    pub third: u32,
+    pub img_size: u32,
+}
+
+#[derive(Deserialize, Debug, Clone)]
+pub struct PersistMkdir {
+    pub name: Vec<String>,
+}
+
+// pxe config
+#[derive(Debug, Deserialize, Clone)]
+pub struct PxeConfig {
+    pub rootfs_name: String,
+    pub disk: String,
+    pub server_ip: String,
+    pub local_ip: String,
+    pub route_ip: String,
+    pub netmask: String,
+    pub net_name: String,
+}
diff --git a/KubeOS-Rust/kbimg/src/docker_img.rs b/KubeOS-Rust/kbimg/src/docker_img.rs
new file mode 100644
index 0000000000000000000000000000000000000000..9c5a8c20f4e07fc9a1de17c05cc818c6e46038c3
--- /dev/null
+++ b/KubeOS-Rust/kbimg/src/docker_img.rs
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{fs::File, path::PathBuf, process::Command, str};
+
+use anyhow::bail;
+
+use crate::{
+    commands::DockerInfo,
+    scripts_gen::*,
+    utils::{self, set_permissions},
+    values::*,
+    Config, CreateImage,
+};
+
+impl CreateImage for DockerInfo {
+    fn prepare(&self, _: &mut Config) -> anyhow::Result<()> {
+        let image_name = &self.docker_img;
+        verify_docker_input(&image_name)?;
+        check_docker_image(&image_name)?;
+        Ok(())
+    }
+
+    fn generate_scripts(&self, config: &Config) -> anyhow::Result<PathBuf> {
+        // kbimg.sh
+        let kbimg_path = format!("{}/{}", SCRIPTS_DIR, KBIMG_SH);
+        let mut kbimg = File::create(&kbimg_path)?;
+        gen_global_vars(&mut kbimg)?;
+        gen_docker_vars(&mut kbimg, &self.docker_img)?;
+        gen_global_func(&mut kbimg)?;
+        gen_create_os_tar_from_docker(&mut kbimg)?;
+        if self.image_type == "vm-docker" {
+            // kbimg.sh
+            gen_init_part(&mut kbimg)?;
+            gen_create_img(&mut kbimg, false, &config)?;
+            gen_create_vm_docker_img(&mut kbimg)?;
+        } else {
+            // kbimg.sh
+            gen_create_pxe_docker_img(&mut kbimg)?;
+        }
+        set_permissions(&kbimg_path, EXEC_PERMISSION)?;
+
+        Ok(PathBuf::from(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH)))
+    }
+}
+
+fn verify_docker_input(image_name: &str) -> anyhow::Result<()> {
+    if !utils::is_valid_param(image_name) {
+        bail!("params {} is invalid, please check input", image_name);
+    }
+    Ok(())
+}
+
+fn check_docker_image(image_name: &str) -> anyhow::Result<()> {
+    let output =
+        Command::new("docker").args(&["images", "-q", image_name]).output().expect("Failed to execute command");
+
+    if output.status.success() {
+        let stdout = str::from_utf8(&output.stdout).expect("Invalid UTF-8 output");
+        if stdout.trim().is_empty() {
+            bail!("docker image does NOT exist, please pull {} first.", image_name);
+        }
+    }
+    Ok(())
+}
diff --git a/KubeOS-Rust/kbimg/src/main.rs b/KubeOS-Rust/kbimg/src/main.rs
new file mode 100644
index 0000000000000000000000000000000000000000..beb389d5d2cb0c74809abe8093767c382e1b1350
--- /dev/null
+++ b/KubeOS-Rust/kbimg/src/main.rs
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{fs, path::PathBuf, process::exit};
+
+use anyhow::{bail, Result};
+use clap::Parser;
+use env_logger::{Builder, Env, Target};
+use log::{debug, error, info};
+
+mod admin_container;
+mod commands;
+mod docker_img;
+mod repo;
+mod scripts_gen;
+mod utils;
+mod values;
+
+use utils::{execute_scripts, get_arch};
+use values::SCRIPTS_DIR;
+
+use crate::commands::{Cli, Commands, Config};
+
+trait CreateImage {
+    /// validate cmd args, check disk size and other prepare work
+    fn prepare(&self, config: &mut Config) -> Result<()>;
+    /// generate scripts for creating image. If debug is enabled, keep the scripts, otherwise execute them
+    fn generate_scripts(&self, config: &Config) -> Result<PathBuf>;
+}
+
+fn process(info: Box<dyn CreateImage>, mut config: Config) -> Result<()> {
+    match fs::create_dir_all(SCRIPTS_DIR) {
+        Ok(_) => {
+            info.prepare(&mut config)?;
+            let path = info.generate_scripts(&config)?;
+            execute_scripts(path)?;
+            Ok(())
+        },
+        Err(e) => bail!(e),
+    }
+}
+
+fn main() {
+    let cli = Cli::parse();
+    let default_log_level: &str = if cli.debug { "debug" } else { "info" };
+    Builder::from_env(Env::default().default_filter_or(default_log_level)).target(Target::Stdout).init();
+    match cli.config {
+        Some(config) => {
+            info!("Loading config file");
+            debug!("Config file path: {:?}", config);
+            let content = fs::read_to_string(config).unwrap();
+            let data: Config = toml::from_str(&content).unwrap();
+            debug!("Config: {:?}", data);
+            let info = if let Some(mut info) = data.from_repo.clone() {
+                info.arch = Some(get_arch());
+                Some(Box::new(info) as Box<dyn CreateImage>)
+            } else if let Some(info) = data.from_dockerimg.clone() {
+                Some(Box::new(info) as Box<dyn CreateImage>)
+            } else if let Some(info) = data.admin_container.clone() {
+                Some(Box::new(info) as Box<dyn CreateImage>)
+            } else {
+                None
+            };
+            if let Some(i) = info {
+                match process(i, data) {
+                    Ok(_) => {
+                        info!("Image created successfully");
+                    },
+                    Err(e) => {
+                        error!("Failed to create image: {:?}", e);
+                    },
+                }
+            }
+            exit(0);
+        },
+        None => {},
+    }
+    let info = match cli.commands {
+        Some(Commands::UpgradeImage(mut info)) => {
+            info.image_type = "upgrade".to_string();
+            Some(Box::new(info) as Box<dyn CreateImage>)
+        },
+        Some(Commands::VMRepo(mut info)) => {
+            info.image_type = "vm-repo".to_string();
+            debug!("VMRepo: {:?}", info);
+            Some(Box::new(info) as Box<dyn CreateImage>)
+        },
+        Some(Commands::VMDocker(mut info)) => {
+            info.image_type = "vm-docker".to_string();
+            Some(Box::new(info) as Box<dyn CreateImage>)
+        },
+        Some(Commands::PxeRepo(mut info)) => {
+            info.image_type = "pxe-repo".to_string();
+            Some(Box::new(info) as Box<dyn CreateImage>)
+        },
+        Some(Commands::PxeDocker(mut info)) => {
+            info.image_type = "pxe-docker".to_string();
+            Some(Box::new(info) as Box<dyn CreateImage>)
+        },
+        Some(Commands::AdminContainer(info)) => Some(Box::new(info) as Box<dyn CreateImage>),
+        None => None,
+    };
+    if let Some(i) = info {
+        match process(i, Config::default()) {
+            Ok(_) => {
+                info!("Image created successfully");
+            },
+            Err(e) => {
+                error!("Failed to create image: {:?}", e);
+            },
+        }
+    }
+}
diff --git a/KubeOS-Rust/kbimg/src/repo.rs b/KubeOS-Rust/kbimg/src/repo.rs
new file mode 100644
index 0000000000000000000000000000000000000000..43aabd3bf778f37d23f6a8ed8771edd1eac7d279
--- /dev/null
+++ b/KubeOS-Rust/kbimg/src/repo.rs
@@ -0,0 +1,203 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{
+    env,
+    fs::{create_dir_all, File},
+    path::PathBuf,
+};
+
+use anyhow::bail;
+use sysinfo::{System, SystemExt, DiskExt};
+
+use crate::{
+    commands::RepoInfo,
+    scripts_gen::*,
+    utils::{self, check_pxe_conf_valid, set_permissions},
+    values::*,
+    Config, CreateImage,
+};
+
+impl CreateImage for RepoInfo {
+    fn prepare(&self, config: &mut Config) -> anyhow::Result<()> {
+        verify_repo_input(&self)?;
+        check_disk_space(&self.image_type)?;
+        check_repo_file_valid(&self.repo_path)?;
+        check_agent_file_valid(&self.agent_path)?;
+        if self.image_type == "pxe-repo" {
+            if let Some(pxe_config) = &config.pxe_config {
+                check_pxe_conf_valid(&pxe_config)?;
+            } else {
+                bail!("pxe config not found!")
+            }
+        }
+        Ok(())
+    }
+
+    fn generate_scripts(&self, config: &Config) -> anyhow::Result<PathBuf> {
+        // rpmlist
+        let rpmlist_path = format!("{}/{}", SCRIPTS_DIR, RPMLIST);
+        let mut rpmlist = File::create(&rpmlist_path)?;
+        gen_rpm_list(&mut rpmlist, &self.rpmlist)?;
+        set_permissions(&rpmlist_path, CONFIG_PERMISSION)?;
+        // 00bootup
+        match create_dir_all(BOOTUP_DIR) {
+            Ok(_) => {
+                if let Some(pxe_config) = &config.pxe_config {
+                    let global_cfg_path = format!("{}/{}", BOOTUP_DIR, BOOTUP_GLOBAL_CFG);
+                    let mut global_cfg = File::create(&global_cfg_path)?;
+                    gen_global_cfg(&mut global_cfg, &pxe_config)?;
+                    set_permissions(&global_cfg_path, CONFIG_PERMISSION)?;
+                }
+                let module_setup_path = format!("{}/{}", BOOTUP_DIR, BOOTUP_MODULE_SETUP_SH);
+                let mut module_setup = File::create(&module_setup_path)?;
+                gen_module_setup(&mut module_setup)?;
+                set_permissions(&module_setup_path, EXEC_PERMISSION)?;
+                let mount_path = format!("{}/{}", BOOTUP_DIR, BOOTUP_MOUNT_SH);
+                let mut mount = File::create(&mount_path)?;
+                gen_mount(&mut mount)?;
+                set_permissions(&mount_path, EXEC_PERMISSION)?;
+            },
+            Err(e) => {
+                bail!(e);
+            },
+        }
+        // misc-files
+        match create_dir_all(MISC_FILES_DIR) {
+            Ok(_) => {
+                let boot_efi_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_BOOT_EFI_MOUNT);
+                let mut boot_efi_mount = File::create(&boot_efi_mount_path)?;
+                gen_boot_efi_mount(&mut boot_efi_mount)?;
+                set_permissions(&boot_efi_mount_path, CONFIG_PERMISSION)?;
+                let boot_grub2_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_BOOT_GRUB2_MOUNT);
+                let mut boot_grub2_mount = File::create(&boot_grub2_mount_path)?;
+                gen_boot_grub2_mount(&mut boot_grub2_mount)?;
+                set_permissions(&boot_grub2_mount_path, CONFIG_PERMISSION)?;
+                let etc_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_ETC_MOUNT);
+                let mut etc_mount = File::create(&etc_mount_path)?;
+                gen_etc_mount(&mut etc_mount)?;
+                set_permissions(&etc_mount_path, CONFIG_PERMISSION)?;
+                let os_agent_service_path = format!("{}/{}", MISC_FILES_DIR, MISC_OS_AGENT_SERVICE);
+                let mut os_agent_service = File::create(&os_agent_service_path)?;
+                gen_os_agent_service(&mut os_agent_service)?;
+                set_permissions(&os_agent_service_path, CONFIG_PERMISSION)?;
+                let os_release_path = format!("{}/{}", MISC_FILES_DIR, MISC_OS_RELEASE);
+                let mut os_release = File::create(&os_release_path)?;
+                gen_os_release(&mut os_release)?;
+                set_permissions(&os_release_path, CONFIG_PERMISSION)?;
+                let persist_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_PERSIST_MOUNT);
+                let mut persist_mount = File::create(&persist_mount_path)?;
+                gen_persist_mount(&mut persist_mount)?;
+                set_permissions(&persist_mount_path, CONFIG_PERMISSION)?;
+                let var_mount_path = format!("{}/{}", MISC_FILES_DIR, MISC_VAR_MOUNT);
+                let mut var_mount = File::create(&var_mount_path)?;
+                gen_var_mount(&mut var_mount)?;
+                set_permissions(&var_mount_path, CONFIG_PERMISSION)?;
+            },
+            Err(e) => {
+                bail!(e);
+            },
+        }
+        // grub.cfg
+        let grub_cfg_path = format!("{}/{}", SCRIPTS_DIR, GRUB_CFG);
+        let mut grub_cfg = File::create(&grub_cfg_path)?;
+        gen_grub_cfg(&mut grub_cfg)?;
+        set_permissions(&grub_cfg_path, CONFIG_PERMISSION)?;
+        // set_in_chroot.sh
+        let set_in_chroot_path = format!("{}/{}", SCRIPTS_DIR, SET_IN_CHROOT_SH);
+        let mut set_in_chroot = File::create(&set_in_chroot_path)?;
+        gen_set_in_chroot(&mut set_in_chroot, self.legacy_bios, &config)?;
+        set_permissions(&set_in_chroot_path, EXEC_PERMISSION)?;
+        // kbimg.sh
+        let kbimg_path = format!("{}/{}", SCRIPTS_DIR, KBIMG_SH);
+        let mut kbimg = File::create(&kbimg_path)?;
+        gen_global_vars(&mut kbimg)?;
+        gen_repo_vars(&mut kbimg, &self)?;
+        gen_global_func(&mut kbimg)?;
+        gen_mount_proc_dev_sys(&mut kbimg)?;
+        gen_unmount_dir(&mut kbimg)?;
+        gen_create_os_tar_from_repo(&mut kbimg, &self, &config)?;
+        if self.image_type == "vm-repo" {
+            // bootloader.sh
+            let bootloader_path = format!("{}/{}", SCRIPTS_DIR, BOOTLOADER_SH);
+            let mut bootloader = File::create(&bootloader_path)?;
+            gen_bootloader(&mut bootloader, self.arch.as_ref().unwrap(), self.legacy_bios)?;
+            set_permissions(&bootloader_path, EXEC_PERMISSION)?;
+            // kbimg.sh
+            gen_init_part(&mut kbimg)?;
+            gen_create_img(&mut kbimg, self.legacy_bios, &config)?;
+            gen_create_vm_repo_img(&mut kbimg)?;
+        } else if self.image_type == "pxe-repo" {
+            // kbimg.sh
+            gen_create_pxe_repo_img(&mut kbimg)?;
+        } else {
+            // Dockerfile
+            let dockerfile_path = format!("{}/{}", SCRIPTS_DIR, DOCKERFILE);
+            let mut dockerfile = File::create(&dockerfile_path)?;
+            gen_dockerfile(&mut dockerfile)?;
+            set_permissions(&dockerfile_path, CONFIG_PERMISSION)?;
+            // kbimg.sh
+            gen_create_docker_img(&mut kbimg)?;
+        }
+        set_permissions(&kbimg_path, EXEC_PERMISSION)?;
+
+        Ok(PathBuf::from(&format!("{}/{}", SCRIPTS_DIR, KBIMG_SH)))
+    }
+}
+
+fn verify_repo_input(info: &RepoInfo) -> anyhow::Result<()> {
+    if !utils::is_valid_param(info.repo_path.to_str().unwrap()) {
+        bail!("params {} is invalid, please check input", info.repo_path.to_str().unwrap());
+    }
+    if !utils::is_valid_param(&info.version) {
+        bail!("params {} is invalid, please check input", info.version);
+    }
+    if !utils::is_valid_param(info.agent_path.to_str().unwrap()) {
+        bail!("params {} is invalid, please check input", info.agent_path.to_str().unwrap());
+    }
+    if let Some(docker_img) = &info.docker_img {
+        if !utils::is_valid_param(docker_img) {
+            bail!("params {} is invalid, please check input", docker_img);
+        }
+    }
+    Ok(())
+}
+
+fn check_disk_space(image_type: &str) -> anyhow::Result<()> {
+    let max_size: u64 = match image_type {
+        "upgrade" => 6,
+        "vm-repo" => 25,
+        "pxe-repo" => 5,
+        _ => bail!("Invalid image type: {}", image_type),
+    };
+
+    let current_dir = env::current_dir().expect("Failed to get current directory");
+    let root_dir = current_dir.ancestors().last().expect("Failed to get current directory").to_path_buf();
+    let mut sys = System::new_all();
+    sys.refresh_all();
+    for d in sys.disks() {
+        if d.mount_point() == root_dir {
+            if d.available_space() < max_size * 1024 * 1024 {
+                bail!("The available disk space is not enough, at least {}GiB.", max_size);
+            }
+        }
+    }
+    Ok(())
+}
+
+fn check_repo_file_valid(repo_path: &PathBuf) -> anyhow::Result<()> {
+    utils::is_file_valid("REPO file", repo_path)
+}
+
+fn check_agent_file_valid(agent_path: &PathBuf) -> anyhow::Result<()> {
+    utils::is_file_valid("os-agent binary", agent_path)
+}
diff --git a/KubeOS-Rust/kbimg/src/scripts_gen.rs b/KubeOS-Rust/kbimg/src/scripts_gen.rs
new file mode 100644
index 0000000000000000000000000000000000000000..7d5748887d1d14e870365b3d6e4beab1e23fd20c
--- /dev/null
+++ b/KubeOS-Rust/kbimg/src/scripts_gen.rs
@@ -0,0 +1,1754 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{fs::File, io::Write, path::PathBuf};
+
+use anyhow::{bail, Ok, Result};
+
+use crate::{commands::*, values::SCRIPTS_DIR};
+
+/* copyright */
+pub(crate) fn gen_copyright(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"## Copyright (c) Huawei Technologies Co., Ltd. 2022. All rights reserved.
+# KubeOS is licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+#     http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+## See the Mulan PSL v2 for more details.
+"#
+    )?;
+
+    Ok(())
+}
+
+/* region: kbimg.sh */
+pub(crate) fn gen_global_vars(file: &mut File) -> Result<()> {
+    writeln!(file, "#!/bin/bash")?;
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"set -e
+
+SCRIPTS_DIR=$(cd "$(dirname "$0")" && pwd)
+LOCK="${{SCRIPTS_DIR}}"/test.lock
+RPM_ROOT="${{SCRIPTS_DIR}}"/rootfs
+TMP_MOUNT_PATH="${{SCRIPTS_DIR}}"/mnt
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_global_func(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function delete_dir() {{
+	local ret=0
+	local dir="$1"
+	unmount_dir "${{dir}}"
+	ret=$?
+	if [ "${{ret}}" -eq 0 ]; then
+        rm -rf "${{dir}}"
+        return 0
+	else
+        log_error_print "${{dir}} is failed to unmount , can not delete ${{dir}}."
+        return 1
+	fi
+}}
+
+function delete_file() {{
+	local file="$1"
+	if [ ! -e "${{file}}" ]; then
+        return 0
+	fi
+
+	if [ ! -f "${{file}}" ]; then
+        log_error_print "${{file}} is not a file."
+        return 1
+	fi
+
+	rm -f "${{file}}"
+	return 0
+}}
+
+function clean_space() {{
+    delete_dir "${{RPM_ROOT}}"
+    delete_dir "${{TMP_MOUNT_PATH}}"
+    delete_file "${{SCRIPTS_DIR}}"/os.tar
+    rm -rf "${{LOCK}}"
+    delete_file "${{ADMIN_CONTAINER_DIR}}"/hostshell
+}}
+
+function clean_img() {{
+    delete_file "${{SCRIPTS_DIR}}"/system.img
+    delete_file "${{SCRIPTS_DIR}}"/update.img
+    delete_file "${{SCRIPTS_DIR}}"/initramfs.img
+    delete_file "${{SCRIPTS_DIR}}"/kubeos.tar
+}}
+
+function file_lock() {{
+    local lock_file=$1
+    exec {{lock_fd}}>"${{lock_file}}"
+    flock -xn "${{lock_fd}}"
+}}
+
+function test_lock() {{
+    file_lock "${{LOCK}}"
+    local status=$?
+    if [ $status -ne 0 ]; then
+        log_error_print "There is already an generate process running."
+        exit 203
+    fi
+}}
+
+function log_error_print() {{
+    local logmsg
+    logmsg="[ ERROR ] - ""$(date "+%b %d %Y %H:%M:%S")"" $1"
+    echo "$logmsg"
+}}
+
+function log_info_print() {{
+    local logmsg
+    logmsg="[ INFO ] - ""$(date "+%b %d %Y %H:%M:%S")"" $1"
+    echo "$logmsg"
+}}
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_mount_proc_dev_sys(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function mount_proc_dev_sys() {{
+	local tmp_root=$1
+	mount -t proc none "${{tmp_root}}"/proc
+	mount --bind /dev "${{tmp_root}}"/dev
+	mount --bind /dev/pts "${{tmp_root}}"/dev/pts
+	mount -t sysfs none "${{tmp_root}}"/sys
+}}
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_unmount_dir(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function unmount_dir() {{
+	local dir=$1
+
+	if [ -L "${{dir}}" ] || [ -f "${{dir}}" ]; then
+        log_error_print "${{dir}} is not a directory, please check it."
+        return 1
+	fi
+
+	if [ ! -d "${{dir}}" ]; then
+        return 0
+	fi
+
+	local real_dir
+	real_dir=$(readlink -e "${{dir}}")
+	local mnts
+	mnts=$(awk '{{print $2}}' < /proc/mounts | grep "^${{real_dir}}" | sort -r)
+	for m in ${{mnts}}; do
+        log_info_print "Unmount ${{m}}"
+        umount -f "${{m}}" || true
+	done
+
+	return 0
+}}
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_init_part(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function init_part() {{
+	local offset
+	offset=$(fdisk -l "${{SCRIPTS_DIR}}"/system.img | grep "$1" | awk '{{print $2}}')
+	local sizelimit
+	sizelimit=$(fdisk -l "${{SCRIPTS_DIR}}"/system.img | grep "$1" | awk '{{print $3}}')
+	sizelimit=$(echo "($sizelimit - $offset)*512" | bc)
+	offset=$(echo "${{offset}}*512" | bc)
+	local loop
+	loop=$(losetup -f)
+	losetup -o "${{offset}}" --sizelimit "${{sizelimit}}" "${{loop}}" "${{SCRIPTS_DIR}}"/system.img
+	if [ "$2" == "BOOT" ];then
+        mkfs.vfat -n "$2" "${{loop}}"
+        mount -t vfat "${{loop}}" "$3"
+	else
+        mkfs.ext4 -L "$2" "${{loop}}"
+        mount -t ext4 "${{loop}}" "$3"
+        rm -rf "$3/lost+found"
+	fi
+}}
+"#
+    )?;
+    Ok(())
+}
+
+// repo
+pub(crate) fn gen_repo_vars(file: &mut File, info: &RepoInfo) -> Result<()> {
+    writeln!(
+        file,
+        r#"REPO_PATH="{}"
+VERSION="{}"
+AGENT_PATH="{}"
+ROOT_PASSWD='{}'
+BOOT_MODE="{}"
+"#,
+        info.repo_path.to_str().unwrap(),
+        &info.version,
+        info.agent_path.to_str().unwrap(),
+        &info.root_passwd,
+        if info.legacy_bios { "legacy" } else { "efi" }
+    )?;
+    if let Some(docker_img) = &info.docker_img {
+        writeln!(file, "DOCKER_IMG=\"{}\"\n", docker_img)?;
+    }
+    Ok(())
+}
+
+pub(crate) fn gen_prepare_yum(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function prepare_yum() {{
+    # init rpmdb
+    rpm --root "${{RPM_ROOT}}" --initdb
+    mkdir -p "${{RPM_ROOT}}"{{/etc/yum.repos.d,/persist,/proc,/dev/pts,/sys}}
+    mount_proc_dev_sys "${{RPM_ROOT}}"
+    # init yum repo
+    local iso_repo="${{RPM_ROOT}}"/etc/yum.repos.d/iso.repo
+    cat "${{REPO_PATH}}" > "$iso_repo"
+}}
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_install_packages(file: &mut File, arch: &str, legacy_bios: bool) -> Result<()> {
+    writeln!(
+        file,
+        r#"function install_packages() {{
+    prepare_yum "${{REPO_PATH}}"
+
+    echo "install package.."
+
+    local filesize
+    filesize=$(stat -c "%s" "${{SCRIPTS_DIR}}"/rpmlist)
+    local maxsize=$((1024*1024))
+    if [ "${{filesize}}" -gt "${{maxsize}}" ]; then
+        echo "please check if rpmlist is too big or something wrong"
+        exit 7
+    fi
+
+    local rpms_name
+    rpms_name=$(tr "\n" " " < "${{SCRIPTS_DIR}}"/rpmlist)
+    old_ifs="$IFS"
+    IFS=' '"#
+    )?;
+
+    if arch == "x86_64" {
+        if legacy_bios {
+            writeln!(file, "\trpms_name+=\" grub2\"")?;
+        } else {
+            writeln!(file, "\trpms_name+=\" grub2-efi grub2-tools grub2-efi-x64-modules grub2-pc-modules\"")?;
+        }
+        writeln!(
+            file,
+            r#"    read -ra rpms <<< "${{rpms_name}}"
+    IFS="$old_ifs"
+    yum -y --installroot="${{RPM_ROOT}}" install --nogpgcheck --setopt install_weak_deps=False "${{rpms[@]}}""#
+        )?;
+    } else if arch == "aarch64" {
+        writeln!(
+            file,
+            r#"        read -ra rpms <<< "${{rpms_name}}"
+        IFS="$old_ifs"
+        yum -y --installroot="${{RPM_ROOT}}" install --nogpgcheck --setopt install_weak_deps=False "${{rpms[@]}}" grub2-efi grub2-tools grub2-efi-aa64-modules"#
+        )?;
+    }
+    writeln!(
+        file,
+        r#"    yum -y --installroot="${{RPM_ROOT}}" clean all
+}}
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_copy_files(file: &mut File, copy_files: &Vec<CopyFile>) -> Result<()> {
+    writeln!(file, "function copy_files() {{")?;
+    for copy_file in copy_files {
+        let dst = format!("{}/rootfs{}", SCRIPTS_DIR, &copy_file.dst);
+        let dst = PathBuf::from(dst);
+        if !dst.exists() {
+            writeln!(file, "\tmkdir -p \"${{RPM_ROOT}}{}\"", &copy_file.dst)?;
+        }
+        let src = PathBuf::from(&copy_file.src);
+        if src.is_dir() {
+            writeln!(file, "\tcp -r {} \"${{RPM_ROOT}}{}\"", &copy_file.src, &copy_file.dst)?;
+        } else {
+            writeln!(file, "\tcp {} \"${{RPM_ROOT}}{}\"", &copy_file.src, &copy_file.dst)?;
+        }
+    }
+    writeln!(file, "}}\n")?;
+    Ok(())
+}
+
+pub(crate) fn gen_grub_config(file: &mut File, legacy_bios: bool, grub: &Grub) -> Result<()> {
+    writeln!(
+        file,
+        r#"function grub_config() {{
+    local GRUB_PATH"#
+    )?;
+    if legacy_bios {
+        writeln!(file, "\tGRUB_PATH=\"${{RPM_ROOT}}\"/boot/grub2")?;
+    } else {
+        writeln!(file, "\tGRUB_PATH=\"${{RPM_ROOT}}\"/boot/efi/EFI/openEuler")?;
+    }
+    if let Some(grub_passwd) = &grub.passwd {
+        writeln!(
+            file,
+            r#"    local GRUB_PASSWD
+    GRUB_PASSWD=$(echo -e "{}\n{}" | grub2-mkpasswd-pbkdf2 | grep PBKDF2 | awk '{{print $7}}')
+    echo "GRUB2_PASSWD=${{GRUB_PASSWD}}" > "${{GRUB_PATH}}"/user.cfg
+    chmod 600 "${{GRUB_PATH}}"/user.cfg
+}}
+"#,
+            grub_passwd, grub_passwd
+        )?;
+    }
+    Ok(())
+}
+
+pub(crate) fn gen_chroot_script(file: &mut File, chroot_script: &ChrootScript) -> Result<()> {
+    let script_path = PathBuf::from(&chroot_script.path);
+    match script_path.canonicalize() {
+        core::result::Result::Ok(absolute_path) => {
+            if let Some(script_name) = absolute_path.file_name() {
+                writeln!(
+                    file,
+                    r#"function chroot_script() {{
+    cp "{}" "${{RPM_ROOT}}"
+    chroot "${{RPM_ROOT}}" bash /{}
+}}
+"#,
+                    absolute_path.as_path().to_str().unwrap(),
+                    script_name.to_str().unwrap()
+                )?;
+            }
+            Ok(())
+        },
+        Err(e) => bail!(e),
+    }
+}
+
+pub(crate) fn gen_install_misc(file: &mut File, legacy_bios: bool, config: &Config) -> Result<()> {
+    if let Some(copy_files) = &config.copy_files {
+        gen_copy_files(file, &copy_files)?;
+    }
+    if let Some(grub) = &config.grub {
+        gen_grub_config(file, legacy_bios, &grub)?;
+    }
+    if let Some(chroot_script) = &config.chroot_script {
+        gen_chroot_script(file, &chroot_script)?;
+    }
+
+    writeln!(
+        file,
+        r#"function install_misc() {{
+    cp "${{SCRIPTS_DIR}}"/misc-files/*mount "${{SCRIPTS_DIR}}"/misc-files/os-agent.service "${{RPM_ROOT}}"/usr/lib/systemd/system/
+    cp "${{SCRIPTS_DIR}}"/misc-files/os-release "${{RPM_ROOT}}"/usr/lib/
+    cp "${{AGENT_PATH}}" "${{RPM_ROOT}}"/usr/bin
+    rm "${{RPM_ROOT}}"/etc/os-release
+
+    cat <<EOF > "${{RPM_ROOT}}"/usr/lib/os-release
+NAME=${{NAME}}
+ID=${{NAME}}
+EOF
+    echo "PRETTY_NAME=\"${{NAME}} ${{VERSION}}\"" >> "${{RPM_ROOT}}"/usr/lib/os-release
+    echo "VERSION_ID=${{VERSION}}" >> "${{RPM_ROOT}}"/usr/lib/os-release
+    mv "${{RPM_ROOT}}"/boot/vmlinuz* "${{RPM_ROOT}}"/boot/vmlinuz
+    mv "${{RPM_ROOT}}"/boot/initramfs* "${{RPM_ROOT}}"/boot/initramfs.img"#
+    )?;
+
+    if legacy_bios {
+        writeln!(
+            file,
+            r#"    cp "${{SCRIPTS_DIR}}"/grub.cfg "${{RPM_ROOT}}"/boot/grub2
+    sed -i "s/insmod part_gpt/insmod part_msdos/g; \
+s/set root='hd0,gpt2'/set root='hd0,msdos2'/g; \
+s/set root='hd0,gpt3'/set root='hd0,msdos3'/g" \
+"${{RPM_ROOT}}"/boot/grub2/grub.cfg"#
+        )?;
+    } else {
+        writeln!(file, "\tcp \"${{SCRIPTS_DIR}}\"/grub.cfg \"${{RPM_ROOT}}\"/boot/efi/EFI/openEuler")?;
+    }
+
+    writeln!(
+        file,
+        r#"    cp -r "${{SCRIPTS_DIR}}"/00bootup "${{RPM_ROOT}}"/usr/lib/dracut/modules.d/ 
+    cp "${{SCRIPTS_DIR}}"/set_in_chroot.sh "${{RPM_ROOT}}"
+    
+    # (optional) custom config"#
+    )?;
+
+    if let Some(_) = &config.copy_files {
+        writeln!(file, "\tcopy_files")?;
+    }
+    if let Some(_) = &config.grub {
+        writeln!(file, "\tgrub_config")?;
+    }
+    if let Some(_) = &config.chroot_script {
+        writeln!(file, "\tchroot_script")?;
+    }
+
+    writeln!(
+        file,
+        r#"
+    ROOT_PASSWD="${{ROOT_PASSWD}}" BOOT_MODE="${{BOOT_MODE}}" chroot "${{RPM_ROOT}}" bash /set_in_chroot.sh
+    rm "${{RPM_ROOT}}/set_in_chroot.sh"
+}}
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_create_os_tar_from_repo(file: &mut File, info: &RepoInfo, config: &Config) -> Result<()> {
+    gen_prepare_yum(file)?;
+    gen_install_packages(file, info.arch.as_ref().unwrap(), info.legacy_bios)?;
+    gen_install_misc(file, info.legacy_bios, config)?;
+
+    writeln!(
+        file,
+        r#"function create_os_tar_from_repo() {{
+    install_packages
+    install_misc
+    unmount_dir "${{RPM_ROOT}}"
+    tar -C "${{RPM_ROOT}}" -cf "${{SCRIPTS_DIR}}"/os.tar .
+}}
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_create_img(file: &mut File, legacy_bios: bool, config: &Config) -> Result<()> {
+    let (first, second, third, img_size) = if let Some(disk_partition) = &config.disk_partition {
+        let first = disk_partition.first;
+        let second = disk_partition.second;
+        let third = disk_partition.third;
+        let img_size = disk_partition.img_size;
+        if first + second + third + 2100 > img_size * 1024 {
+            bail!("Image size({}G) is not enough for partitions, please check input", img_size)
+        }
+        (first, first + second, first + second + third, img_size)
+    } else {
+        (60, 2160, 4260, 20)
+    };
+
+    writeln!(
+        file,
+        r#"function create_img() {{
+    rm -f "${{SCRIPTS_DIR}}"/system.img "${{SCRIPTS_DIR}}/update.img"
+    qemu-img create "${{SCRIPTS_DIR}}/system.img" {}G"#,
+        img_size
+    )?;
+
+    if legacy_bios {
+        writeln!(
+            file,
+            r#"    local BOOT_PATH=${{TMP_MOUNT_PATH}}/boot/grub2
+    parted "${{SCRIPTS_DIR}}/system.img" -s mklabel msdos
+    parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 1MiB {}MiB"#,
+            first
+        )?;
+    } else {
+        writeln!(
+            file,
+            r#"    local BOOT_PATH=${{TMP_MOUNT_PATH}}/boot/efi
+    parted "${{SCRIPTS_DIR}}/system.img" -s mklabel gpt
+    parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary fat32 1MiB {}MiB"#,
+            first
+        )?;
+    }
+
+    writeln!(
+        file,
+        r#"    parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 {}MiB {}MiB
+    parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 {}MiB {}MiB
+    parted "${{SCRIPTS_DIR}}/system.img" -s mkpart primary ext4 {}MiB 100%"#,
+        first, second, second, third, third
+    )?;
+
+    writeln!(
+        file,
+        r#"    local device
+    device=$(losetup -f)
+    losetup "${{device}}" "${{SCRIPTS_DIR}}"/system.img
+
+    mkdir -p "${{TMP_MOUNT_PATH}}"
+
+    init_part "${{SCRIPTS_DIR}}"/system.img2 ROOT-A "${{TMP_MOUNT_PATH}}"
+    
+    mkdir -p "${{BOOT_PATH}}"
+    chmod 755 "${{BOOT_PATH}}""#
+    )?;
+
+    if legacy_bios {
+        writeln!(
+            file,
+            r#"    init_part "${{SCRIPTS_DIR}}"/system.img1 GRUB2 "${{BOOT_PATH}}"
+    tar -x -C "${{TMP_MOUNT_PATH}}" -f "${{SCRIPTS_DIR}}"/os.tar
+    sed -i "s/insmod part_gpt/insmod part_msdos/g; \
+s/set root='hd0,gpt2'/set root='hd0,msdos2'/g; \
+s/set root='hd0,gpt3'/set root='hd0,msdos3'/g" \
+"${{TMP_MOUNT_PATH}}"/boot/grub2/grub.cfg"#
+        )?;
+    } else {
+        writeln!(
+            file,
+            r#"    init_part "${{SCRIPTS_DIR}}"/system.img1 BOOT "${{BOOT_PATH}}"
+    tar -x -C "${{TMP_MOUNT_PATH}}" -f "${{SCRIPTS_DIR}}"/os.tar"#
+        )?;
+    }
+
+    writeln!(
+        file,
+        r#"    sync
+    cp "${{SCRIPTS_DIR}}"/bootloader.sh "${{TMP_MOUNT_PATH}}"
+    mount_proc_dev_sys "${{TMP_MOUNT_PATH}}"
+    DEVICE="${{device}}" BOOT_MODE="${{BOOT_MODE}}" chroot "${{TMP_MOUNT_PATH}}" bash bootloader.sh
+    rm -rf "${{TMP_MOUNT_PATH}}"/bootloader.sh
+    sync
+
+    dd if=/dev/disk/by-label/ROOT-A of="${{SCRIPTS_DIR}}"/update.img bs=8M
+    sync
+    unmount_dir "${{TMP_MOUNT_PATH}}"
+    init_part "${{SCRIPTS_DIR}}"/system.img3 ROOT-B "${{TMP_MOUNT_PATH}}"
+    umount "${{TMP_MOUNT_PATH}}"
+
+    init_part "${{SCRIPTS_DIR}}"/system.img4 PERSIST "${{TMP_MOUNT_PATH}}"
+    mkdir "${{TMP_MOUNT_PATH}}"/{{var,etc,etcwork}}"#
+    )?;
+
+    if let Some(persist_mkdir) = &config.persist_mkdir {
+        for name in &persist_mkdir.name {
+            writeln!(file, "\tmkdir \"${{TMP_MOUNT_PATH}}\"/{}", name)?;
+        }
+    }
+
+    writeln!(
+        file,
+        r#"    mkdir -p "${{TMP_MOUNT_PATH}}"/etc/KubeOS/certs
+    umount "${{TMP_MOUNT_PATH}}"
+
+    losetup -D
+    parted "${{SCRIPTS_DIR}}"/system.img -- set 1 boot on
+    qemu-img convert "${{SCRIPTS_DIR}}"/system.img -O qcow2 "${{SCRIPTS_DIR}}"/system.qcow2
+}}
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_create_vm_repo_img(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function create_vm_repo_img() {{
+	create_os_tar_from_repo
+	create_img
+}}
+    
+test_lock
+trap clean_space EXIT
+trap clean_img ERR
+
+create_vm_repo_img"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_create_pxe_repo_img(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function create_pxe_repo_img() {{
+    rm -rf "${{SCRIPTS_DIR}}"/initramfs.img "${{SCRIPTS_DIR}}"/kubeos.tar
+	create_os_tar_from_repo
+	tar -xvf "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/initramfs.img
+  	mv "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/kubeos.tar
+}}
+
+test_lock
+trap clean_space EXIT
+trap clean_img ERR
+
+create_pxe_repo_img"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_create_docker_img(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function create_docker_img() {{
+	create_os_tar_from_repo
+	docker build -t "${{DOCKER_IMG}}" -f "${{SCRIPTS_DIR}}"/Dockerfile .
+}}
+
+test_lock
+trap clean_space EXIT
+trap clean_img ERR
+
+create_docker_img"#
+    )?;
+    Ok(())
+}
+
+// docker
+pub(crate) fn gen_docker_vars(file: &mut File, image_name: &str) -> Result<()> {
+    writeln!(
+        file,
+        r#"
+IMAGE_NAME="{}"
+BOOT_MODE=efi
+"#,
+        image_name
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_create_os_tar_from_docker(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function create_os_tar_from_docker() {{
+    container_id=$(docker create "${{DOCKER_IMG}}")
+    echo "$container_id"
+    docker cp "$container_id":/os.tar "${{SCRIPTS_DIR}}"
+    docker rm "$container_id"
+}}
+"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_create_vm_docker_img(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function create_vm_docker_img() {{
+    create_os_tar_from_docker
+    create_img
+}}
+
+test_lock
+trap clean_space EXIT
+trap clean_img ERR
+
+create_vm_docker_img"#
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_create_pxe_docker_img(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function create_pxe_docker_img() {{
+    rm -rf "${{SCRIPTS_DIR}}"/initramfs.img "${{SCRIPTS_DIR}}"/kubeos.tar
+    create_os_tar_from_docker
+    tar -xvf "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/initramfs.img
+    mv "${{SCRIPTS_DIR}}"/os.tar "${{SCRIPTS_DIR}}"/kubeos.tar
+}}
+
+test_lock
+trap clean_space EXIT
+trap clean_img ERR
+
+create_pxe_docker_img"#
+    )?;
+    Ok(())
+}
+
+// admin
+pub(crate) fn gen_admin_vars(file: &mut File, docker_img: &str, dockerfile: &PathBuf) -> Result<()> {
+    writeln!(file, "#!/bin/bash")?;
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"set -e
+
+DOCKER_IMG={}
+DOCKERFILE={}
+ADMIN_CONTAINER_DIR="${{SCRIPTS_DIR}}"/admin-container
+"#,
+        dockerfile.to_str().unwrap(),
+        docker_img
+    )?;
+    Ok(())
+}
+
+pub(crate) fn gen_create_admin_img(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"function create_admin_img() {{
+    local kubeos_root_dir=$(dirname $(dirname $(dirname "${{SCRIPTS_DIR}}")))
+    cp "${{kubeos_root_dir}}"/bin/hostshell "${{ADMIN_CONTAINER_DIR}}"
+    docker build -t "${{DOCKER_IMG}}" -f "${{DOCKERFILE}}" "${{ADMIN_CONTAINER_DIR}}"
+    rm -rf "${{ADMIN_CONTAINER_DIR}}"/hostshell
+}}
+
+test_lock
+trap clean_space EXIT
+trap clean_img ERR
+
+create_admin_img"#
+    )?;
+    Ok(())
+}
+/* endregion */
+
+/* region: set_in_chroot.sh */
+pub(crate) fn gen_add_users(file: &mut File, users: &Vec<User>) -> Result<()> {
+    writeln!(file, "# add users")?;
+    for user in users {
+        let name = &user.name;
+        let passwd = &user.passwd;
+        let groups = match user.groups.clone() {
+            Some(groups) => groups,
+            None => vec![name.clone()],
+        };
+        for group in &groups {
+            writeln!(
+                file,
+                r#"if ! getent group "{}" > /dev/null 2>&1; then
+        groupadd "{}"
+fi"#,
+                group, group
+            )?;
+        }
+        write!(file, "useradd -m -g {}", &groups[0])?;
+        if groups.len() > 1 {
+            let additional_groups = &groups[1..].join(",");
+            write!(file, " -G {}", additional_groups)?;
+        }
+        writeln!(file, " -s /bin/bash \"{}\"", &name)?;
+        writeln!(file, "echo \"{}:{}\" | chpasswd", name, passwd)?;
+        if let Some(sudo) = &user.sudo {
+            writeln!(
+                file,
+                r#"if visudo -c; then
+    echo -e "{}    {}" | tee -a /etc/sudoers
+else
+    echo "Sudoers file syntax check failed. Please fix the sudoers file manually."
+    exit 5
+fi"#,
+                name, sudo
+            )?;
+        }
+    }
+    Ok(())
+}
+
+pub(crate) fn gen_systemd_services(file: &mut File, systemd_services: &SystemdService) -> Result<()> {
+    writeln!(file, "# systemd")?;
+    for service_name in &systemd_services.name {
+        writeln!(file, "systemctl enable {}", service_name)?;
+    }
+    Ok(())
+}
+
+pub(crate) fn gen_set_in_chroot(file: &mut File, legacy_bios: bool, config: &Config) -> Result<()> {
+    writeln!(file, "#!/bin/bash")?;
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"ln -s /usr/lib/systemd/system/os-agent.service /usr/lib/systemd/system/multi-user.target.wants/os-agent.service
+ln -s /usr/lib/systemd/system/kubelet.service /usr/lib/systemd/system/multi-user.target.wants/kubelet.service"#
+    )?;
+    if legacy_bios {
+        writeln!(
+            file,
+            "ln -s /usr/lib/systemd/system/boot-grub2.mount /lib/systemd/system/local-fs.target.wants/boot-grub2.mount"
+        )?;
+    } else {
+        writeln!(
+            file,
+            "ln -s /usr/lib/systemd/system/boot-efi.mount /lib/systemd/system/local-fs.target.wants/boot-efi.mount"
+        )?;
+    }
+    writeln!(file, r#"ln -s /usr/lib/systemd/system/etc.mount /lib/systemd/system/local-fs.target.wants/etc.mount"#)?;
+
+    if let Some(users) = &config.users {
+        gen_add_users(file, users)?;
+    }
+    if let Some(systemd_services) = &config.systemd_service {
+        gen_systemd_services(file, systemd_services)?;
+    }
+
+    writeln!(
+        file,
+        r#"
+str=$(sed -n '/^root:/p' /etc/shadow | awk -F "root:" '{{print $2}}')
+umask 0666
+mv /etc/shadow /etc/shadow_bak
+sed -i '/^root:/d' /etc/shadow_bak
+echo "root:""${{ROOT_PASSWD}}""${{str:1}}" > /etc/shadow
+cat /etc/shadow_bak >> /etc/shadow
+rm -rf /etc/shadow_bak
+
+dracut -f -v --add bootup /initramfs.img --kver "$(ls /lib/modules)"
+rm -rf /usr/lib/dracut/modules.d/00bootup"#
+    )?;
+
+    Ok(())
+}
+/* endregion */
+
+/* region: bootloader.sh */
+pub(crate) fn gen_bootloader(file: &mut File, arch: &str, legacy_bios: bool) -> Result<()> {
+    writeln!(file, "#!/bin/bash")?;
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"set -eu
+set -o pipefail
+set -x
+
+function install_grub2 () {{"#
+    )?;
+
+    if arch == "aarch64" || (arch == "x86_64" && !legacy_bios) {
+        writeln!(
+            file,
+            r#"    cp -r /usr/lib/grub/x86_64-efi boot/efi/EFI/openEuler
+    eval "grub2-mkimage -d /usr/lib/grub/x86_64-efi -O x86_64-efi --output=/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux"
+
+    mkdir -p /boot/efi/EFI/BOOT/
+    cp -f /boot/efi/EFI/openEuler/grubx64.efi /boot/efi/EFI/BOOT/BOOTX64.EFI
+}}    
+"#
+        )?;
+    } else {
+        writeln!(
+            file,
+            r#"    GRUBNAME=$(which grub2-install)
+    echo "Installing GRUB2..."
+    FORCE_OPT=${{FORCE_OPT:-"--force"}}
+    TARGET_OPT=${{TARGET_OPT:-"--target=i386-pc"}}
+
+    $GRUBNAME --modules="biosdisk part_msdos" "$FORCE_OPT" "$TARGET_OPT" "$DEVICE"
+}}
+"#
+        )?;
+    }
+
+    writeln!(
+        file,
+        r#"install_grub2
+"#
+    )?;
+    Ok(())
+}
+/* endregion */
+
+/* region: rpmlist */
+pub(crate) fn gen_rpm_list(file: &mut File, rpmlist: &Vec<String>) -> Result<()> {
+    for rpm in rpmlist {
+        writeln!(file, "{}", rpm)?;
+    }
+    Ok(())
+}
+/* endregion */
+
+/* region: grub.cfg */
+pub(crate) fn gen_grub_cfg(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"set pager=1
+
+if [ -f ${{config_directory}}/grubenv ]; then
+  load_env -f ${{config_directory}}/grubenv
+elif [ -s $prefix/grubenv ]; then
+  load_env
+fi
+if [ "${{next_entry}}" ] ; then
+   set default="${{next_entry}}"
+   set next_entry=
+   save_env next_entry
+   set boot_once=true
+else
+   set default="${{saved_entry}}"
+fi
+
+if [ x"${{feature_menuentry_id}}" = xy ]; then
+  menuentry_id_option="--id"
+else
+  menuentry_id_option=""
+fi
+
+export menuentry_id_option
+
+if [ "${{prev_saved_entry}}" ]; then
+  set saved_entry="${{prev_saved_entry}}"
+  save_env saved_entry
+  set prev_saved_entry=
+  save_env prev_saved_entry
+  set boot_once=true
+fi
+
+function savedefault {{
+  if [ -z "${{boot_once}}" ]; then
+    saved_entry="${{chosen}}"
+    save_env saved_entry
+  fi
+}}
+
+function load_video {{
+  if [ x$feature_all_video_module = xy ]; then
+    insmod all_video
+  else
+    insmod efi_gop
+    insmod efi_uga
+    insmod ieee1275_fb
+    insmod vbe
+    insmod vga
+    insmod video_bochs
+    insmod video_cirrus
+  fi
+}}
+
+terminal_output console
+if [ x$feature_timeout_style = xy ] ; then
+  set timeout_style=menu
+  set timeout=5
+# Fallback normal timeout code in case the timeout_style feature is
+# unavailable.
+else
+  set timeout=5
+fi
+set superusers="root"
+### END /etc/grub.d/00_header ###
+
+### BEGIN /etc/grub.d/01_users ###
+if [ -f ${{prefix}}/user.cfg ]; then
+  source ${{prefix}}/user.cfg
+  if [ -n "${{GRUB2_PASSWORD}}" ]; then
+    set superusers="root"
+    export superusers
+    password_pbkdf2 root ${{GRUB2_PASSWORD}}
+  fi
+fi
+### END /etc/grub.d/01_users ###
+
+### BEGIN /etc/grub.d/10_linux ###
+menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' {{
+	load_video
+	set gfxpayload=keep
+	insmod gzio
+	insmod part_gpt
+	insmod ext2
+	set root='hd0,gpt2'
+	linux   /boot/vmlinuz root=/dev/vda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3
+	initrd  /boot/initramfs.img
+}}
+
+menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' {{
+	load_video
+	set gfxpayload=keep
+	insmod gzio
+	insmod part_gpt
+	insmod ext2
+	set root='hd0,gpt3'
+	linux   /boot/vmlinuz root=/dev/vda3 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3
+	initrd  /boot/initramfs.img
+}}
+
+### END /etc/grub.d/10_linux ###
+
+### BEGIN /etc/grub.d/10_reset_boot_success ###
+# Hiding the menu is ok if last boot was ok or if this is a first boot attempt to boot the entry
+if [ "${{boot_success}}" = "1" -o "${{boot_indeterminate}}" = "1" ]; then
+  set menu_hide_ok=1
+else
+  set menu_hide_ok=0
+fi
+# Reset boot_indeterminate after a successful boot
+if [ "${{boot_success}}" = "1" ] ; then
+  set boot_indeterminate=0
+# Avoid boot_indeterminate causing the menu to be hidden more then once
+elif [ "${{boot_indeterminate}}" = "1" ]; then
+  set boot_indeterminate=2
+fi
+# Reset boot_success for current boot
+set boot_success=0
+save_env boot_success boot_indeterminate
+### END /etc/grub.d/10_reset_boot_success ###
+
+### BEGIN /etc/grub.d/12_menu_auto_hide ###
+if [ x$feature_timeout_style = xy ] ; then
+  if [ "${{menu_show_once}}" ]; then
+    unset menu_show_once
+    save_env menu_show_once
+    set timeout_style=menu
+    set timeout=60
+  elif [ "${{menu_auto_hide}}" -a "${{menu_hide_ok}}" = "1" ]; then
+    set orig_timeout_style=${{timeout_style}}
+    set orig_timeout=${{timeout}}
+    if [ "${{fastboot}}" = "1" ]; then
+      # timeout_style=menu + timeout=0 avoids the countdown code keypress check
+      set timeout_style=menu
+      set timeout=0
+    else
+      set timeout_style=hidden
+      set timeout=1
+    fi
+  fi
+fi
+### END /etc/grub.d/12_menu_auto_hide ###
+
+### BEGIN /etc/grub.d/20_linux_xen ###
+### END /etc/grub.d/20_linux_xen ###
+
+### BEGIN /etc/grub.d/20_ppc_terminfo ###
+### END /etc/grub.d/20_ppc_terminfo ###
+
+### BEGIN /etc/grub.d/30_uefi-firmware ###
+### END /etc/grub.d/30_uefi-firmware ###
+
+### BEGIN /etc/grub.d/40_custom ###
+# This file provides an easy way to add custom menu entries.  Simply type the
+# menu entries you want to add after this comment.  Be careful not to change
+# the 'exec tail' line above.
+### END /etc/grub.d/40_custom ###
+
+### BEGIN /etc/grub.d/41_custom ###
+if [ -f  ${{config_directory}}/custom.cfg ]; then
+  source ${{config_directory}}/custom.cfg
+elif [ -z "${{config_directory}}" -a -f  $prefix/custom.cfg ]; then
+  source $prefix/custom.cfg;
+fi
+### END /etc/grub.d/41_custom ###
+"#
+    )?;
+    Ok(())
+}
+/* endregion */
+
+/* region: 00bootup */
+// 00bootup/global.cfg
+pub(crate) fn gen_global_cfg(file: &mut File, pxe_config: &PxeConfig) -> Result<()> {
+    writeln!(
+        file,
+        r#"# rootfs file name
+rootfs_name={}
+
+# select the target disk to install kubeOS
+disk={}
+
+# pxe server ip address where stores the rootfs on the http server
+server_ip={}
+# target machine ip
+local_ip={}
+# target machine route
+route_ip={}
+# target machine netmask
+netmask={}
+# target machine netDevice name
+net_name={}
+"#,
+        pxe_config.rootfs_name,
+        pxe_config.disk,
+        pxe_config.server_ip,
+        pxe_config.local_ip,
+        pxe_config.route_ip,
+        pxe_config.netmask,
+        pxe_config.net_name
+    )?;
+    Ok(())
+}
+
+// 00bootup/module-setup.sh
+pub(crate) fn gen_module_setup(file: &mut File) -> Result<()> {
+    writeln!(file, "#!/bin/bash")?;
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"check() {{
+    return 0
+}}
+
+depends() {{
+    echo systemd
+}}
+
+install() {{
+    inst_multiple -o grub2-mkimage mkfs.ext4 mkfs.vfat lsblk tar cpio gunzip lspci parted dhclient ifconfig curl hwinfo head tee arch df awk route 
+    inst_hook mount 00 "$moddir/mount.sh"
+    inst_simple "$moddir/mount.sh" "/mount.sh"
+    inst_simple "$moddir/Global.cfg" "/Global.cfg"
+}}
+
+installkernel() {{
+    hostonly=''
+    instmods='drivers/ata drivers/nvme drivers/scsi drivers/net fs/fat fs/nls'
+}}
+"#
+    )?;
+    Ok(())
+}
+
+// 00bootup/mount.sh
+pub(crate) fn gen_mount(file: &mut File) -> Result<()> {
+    writeln!(file, "#!/bin/bash")?;
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"arch=$(arch)
+min_size=8
+log=/install.log
+
+source ./Global.cfg
+
+function CheckSpace() {{
+    local disk_ava
+    disk_ava="$(parted -l | grep "${{disk}}" | awk '{{print $3}}')"
+    if echo "${{disk_ava}}" | grep "[GT]B$"; then
+        if echo "${{disk_ava}}" | grep GB$; then
+            disk_ava="$(echo "${{disk_ava}}" | awk -F G '{{print $1}}' | awk -F . '{{print $1}}')"
+            if [ "${{disk_ava}}" -lt ${{min_size}} ]; then
+                echo "The available disk space is not enough, at least ${{min_size}}GB." | tee -a ${{log}}
+                return 1
+            fi
+        fi
+    else
+        echo "The available disk space is not enough, at least ${{min_size}}G." | tee -a ${{log}}
+        return 1
+    fi
+
+    return 0
+}}
+
+function mount_proc_dev_sys() {{
+    local tmp_root=$1
+    mount -t proc none "${{tmp_root}}/proc"
+    mount --bind /dev "${{tmp_root}}/dev"
+    mount --bind /dev/pts "${{tmp_root}}/dev/pts"
+    mount -t sysfs none "${{tmp_root}}/sys"
+}}
+
+function GetDisk() {{
+    mapfile -t disks < <(hwinfo --disk --short 2>&1 | grep -vi "^disk" | awk '{{print $1}}')
+    if [ ${{#disks[*]}} -gt 0 ]; then
+        if [ -n "${{disk}}" ] && echo "${{disks[@]}}" | grep -wq "${{disk}}" ; then
+            echo "${{disk}} exists, start partition"  | tee -a ${{log}}
+        else
+            echo "disk not exist, please choose correct disk"  | tee -a ${{log}}
+        fi
+    else
+        echo "no disk found" | tee -a ${{log}}
+        return 1
+    fi
+    CheckSpace
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "no enough space on ${{disk}}" | tee -a ${{log}}
+        return 1
+    fi
+
+    return 0
+}}
+
+function PartitionAndFormatting() {{
+    echo "Partitioning and formatting disk $disk..."
+    # partition and format
+    parted "${{disk}}" -s mklabel gpt >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "partition failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    parted "${{disk}}" -s mkpart primary fat16 1M 100M >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "partition failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    parted "${{disk}}" -s mkpart primary ext4 100M 2600M >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "partition failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    parted "${{disk}}" -s mkpart primary ext4 2600M 5100M >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "partition failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    parted "${{disk}}" -s mkpart primary ext4 5100M 100% >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "partition failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    parted "${{disk}}" -s set 1 boot on >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "partition failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    mkfs.vfat -n "BOOT" "${{disk}}"1 >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "format failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    mkfs.ext4 -L "ROOT-A" "${{disk}}"2 >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "format failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    mkfs.ext4 -L "ROOT-B" "${{disk}}"3 >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "format failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    mkfs.ext4 -L "PERSIST" "${{disk}}"4 >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "format failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    return 0
+}}
+
+function InitNetwork() {{
+    echo "Initializing network..."
+    mapfile -t netNames < <(ifconfig -a | awk '{{print $1}}' | grep : | grep '^e' | awk -F: '{{print $1}}')
+    if [ ${{#netNames[*]}} -gt 0 ]; then
+        if [ -n "${{net_name}}" ] && echo "${{netNames[@]}}" | grep -wq "${{net_name}}" ; then
+            echo "${{net_name}} exists, start set ip"  | tee -a ${{log}}
+        else
+            echo "net_name not exist, choose default net"  | tee -a ${{log}}
+            net_name=${{netNames[0]}}
+        fi
+    else
+        echo "no net Device found" | tee -a ${{log}}
+        return 1
+    fi
+
+    ifconfig "${{net_name}}" up
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "load net card failed" | tee -a ${{log}}
+        return 1
+    fi
+    sleep 3
+
+    ifconfig "${{net_name}}" "${{local_ip}}" netmask "${{netmask}}"  >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "ip set failed" | tee -a ${{log}}
+        return 1
+    fi
+    sleep 3
+
+    route add default gw "${{route_ip}}" >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "add route failed" | tee -a ${{log}}
+        return 1
+    fi
+    sleep 3
+    return 0
+}}
+
+function MountRoot() {{
+    echo "Mounting rootfs..."
+    # mount rootfs
+    mount "${{disk}}"2 /sysroot >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "mount rootfs failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    return 0
+}}
+
+function MountPersist() {{
+    echo "Mounting persist"
+    mount "${{disk}}"4 /sysroot/persist >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "mount persist failed" | tee -a ${{log}}
+        return 1
+    fi
+    mkdir /sysroot/persist/{{var,etc,etcwork}}
+    mkdir -p /sysroot/persist/etc/KubeOS/certs
+    return 0
+}}
+
+function MountBoot() {{
+    echo "Mounting boot"
+    mkdir -p /sysroot/boot/efi
+    mount "${{disk}}"1 /sysroot/boot/efi >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "mount boot failed" | tee -a ${{log}}
+        return 1
+    fi
+    return 0
+}}
+
+function GetRootfs() {{
+    echo "Downloading rootfs..."
+
+    curl -o /"${{rootfs_name}}" http://"${{server_ip}}"/"${{rootfs_name}}"
+    if [ ! -e "/${{rootfs_name}}" ]; then
+        echo "download rootfs failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    tar -xf /"${{rootfs_name}}" -C /sysroot
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "decompose rootfs failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    rm -rf "${{rootfs_name:?}}"
+    mount -o remount,ro "${{disk}}"2 /sysroot  >> ${{log}} 2>&1
+    return 0
+}}
+
+function Inst_Grub2_x86() {{
+    # copy the files that boot need
+    cp -r /sysroot/usr/lib/grub/x86_64-efi /sysroot/boot/efi/EFI/openEuler
+    eval "grub2-mkimage -d /sysroot/usr/lib/grub/x86_64-efi -O x86_64-efi --output=/sysroot/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux"  >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "grub2-mkimage on x86 failed" | tee -a ${{log}}
+        return 1
+    fi
+    
+    mkdir -p /sysroot/boot/efi/EFI/BOOT/
+    cp -f /sysroot/boot/efi/EFI/openEuler/grubx64.efi /sysroot/boot/efi/EFI/BOOT/BOOTX64.EFI
+
+    return 0
+}}
+
+function Inst_Grub2_aarch64() {{
+    cp -r /sysroot/usr/lib/grub/arm64-efi /sysroot/boot/efi/EFI/openEuler/
+    eval "grub2-mkimage -d /sysroot/usr/lib/grub/arm64-efi -O arm64-efi --output=/sysroot/boot/efi/EFI/openEuler/grubaa64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux"  >> ${{log}} 2>&1
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "grub2-mkimage on aarch64 failed" | tee -a ${{log}}
+        return 1
+    fi
+    
+    mkdir -p /sysroot/boot/efi/EFI/BOOT/
+    cp -f /sysroot/boot/efi/EFI/openEuler/grubaa64.efi /sysroot/boot/efi/EFI/BOOT/BOOTAA64.EFI
+
+    return 0
+}}
+
+function SetBoot() {{
+    # mount boot
+    echo "Setting boot"
+
+    if [ "$arch" == "x86_64" ];   then
+        Inst_Grub2_x86
+        local status=$?
+        if [ $status -ne 0 ]; then
+            echo "install grub on x86 failed" | tee -a ${{log}}
+            return 1
+        fi
+    fi
+
+    if [ "$arch" == "aarch64" ]; then
+        Inst_Grub2_aarch64
+        local status=$?
+        if [ $status -ne 0 ]; then
+            echo "install grub on aarch64 failed" | tee -a ${{log}}
+            return 1
+        fi
+    fi
+    sed -i 's#/dev/sda#'"${{disk}}"'#g' /sysroot/boot/efi/EFI/openEuler/grub.cfg
+
+    return 0
+}}
+
+function Bootup_Main() {{
+    # get disk
+    echo "Checking disk info..." | tee -a ${{log}}
+    GetDisk
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "Checking disk info failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    # partition and format disk
+    echo "Partion and formatting..." | tee -a ${{log}}
+    PartitionAndFormatting
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "Partition and formatting disk failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    # init network
+    echo "Initializing network..." | tee -a ${{log}}
+    InitNetwork
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "Initializing network failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    # mount partitions
+    
+    # mount boot
+    echo "Mounting root..." | tee -a ${{log}}
+    MountRoot
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "Mounting root failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    echo "Mounting boot..." | tee -a ${{log}}
+    MountBoot
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "Mounting boot failed" | tee -a ${{log}}
+        return 1
+    fi
+
+    # download rootfs
+    echo "Downloading rootfs..." | tee -a ${{log}}
+    GetRootfs
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "Downloading rootfs failed" | tee -a ${{log}}
+        return 1
+    fi
+    mount_proc_dev_sys /sysroot
+    # set boot
+    echo "Setting boot..." | tee -a ${{log}}
+    SetBoot
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "Setting boot failed" | tee -a ${{log}}
+        return 1
+    fi
+    # mount persist
+    echo "Mounting persist..." | tee -a ${{log}}
+    MountPersist
+    local status=$?
+    if [ $status -ne 0 ]; then
+        echo "Mounting persist failed" | tee -a ${{log}}
+        return 1
+    fi
+    return 0
+}}
+
+Bootup_Main
+ret=$?
+if [ ${{ret}} -eq 0 ]; then
+    echo "kubeOS install success! switch to root" | tee -a ${{log}}
+    cp ${{log}} /sysroot/persist
+else
+    echo "kubeOS install failed, see install.log" | tee -a ${{log}}
+fi
+
+"#
+    )?;
+    Ok(())
+}
+/* endregion */
+
+/* region: dockerfile */
+pub(crate) fn gen_dockerfile(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"FROM scratch
+COPY os.tar /
+CMD ["/bin/sh"]
+"#
+    )?;
+    Ok(())
+}
+/* endregion */
+
+/* region: admin-container */
+// admin-container/dockerfile
+pub(crate) fn gen_admin_dockerfile(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"FROM openeuler-22.03-lts
+MAINTAINER <shenyangyang4@huawei.com>
+
+RUN yum -y install openssh-clients util-linux
+
+ADD ./sysmaster-0.2.3-1.oe2203.aarch64.rpm /home
+RUN rpm -ivh  /home/sysmaster-0.2.3-1.oe2203.aarch64.rpm
+
+COPY ./hostshell /usr/bin/
+COPY ./set-ssh-pub-key.sh /usr/local/bin
+COPY ./set-ssh-pub-key.service /usr/lib/sysmaster
+
+EXPOSE 22
+# set sshd.service and set-ssh-pub-key.service pulled up by default
+RUN sed -i 's/sysinit.target/sysinit.target;sshd.service;set-ssh-pub-key.service/g' /usr/lib/sysmaster/basic.target
+
+CMD ["/usr/lib/sysmaster/init"]
+"#
+    )?;
+    Ok(())
+}
+
+// admin-container/set-ssh-pub-key.service
+pub(crate) fn gen_set_ssh_pub_key_service(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"[Unit]
+Description="set ssh authorized keys according to the secret which is set by user"
+
+[Service]
+ExecStart="/usr/local/bin/set-ssh-pub-key.sh"
+"#
+    )?;
+    Ok(())
+}
+
+// admin-container/set-ssh-pub-key.sh
+pub(crate) fn gen_set_ssh_pub_key(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"ssh_pub=$(cat /etc/secret-volume/ssh-pub-key)
+ssh_dir="/root/.ssh"
+authorized_file="$ssh_dir/authorized_keys"
+
+if [ ! -d "$ssh_dir" ]; then
+    mkdir "$ssh_dir"
+    chmod 700 "$ssh_dir"
+fi
+
+if [ ! -f "$authorized_file" ]; then
+    touch "$authorized_file"
+    chmod 600 "$authorized_file"
+fi
+
+echo "$ssh_pub" >> "$authorized_file"
+"#
+    )?;
+    Ok(())
+}
+/* endregion */
+
+/* region: misc-files */
+// misc-files/boot-efi.mount
+pub(crate) fn gen_boot_efi_mount(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"[Unit]
+Description=grub2 Dir
+DefaultDependencies=no
+Conflicts=umount.target
+Before=local-fs.target umount.target
+
+[Mount]
+What=/dev/disk/by-label/BOOT
+Where=/boot/efi
+Type=vfat
+Options=defaults
+
+[Install]
+WantedBy=local-fs.target
+"#
+    )?;
+    Ok(())
+}
+
+// misc-files/boot-grub2.mount
+pub(crate) fn gen_boot_grub2_mount(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"[Unit]
+Description=grub2 Dir
+DefaultDependencies=no
+Conflicts=umount.target
+Before=local-fs.target umount.target
+
+[Mount]
+What=/dev/disk/by-label/GRUB2
+Where=/boot/grub2
+Type=ext4
+Options=defaults
+
+[Install]
+WantedBy=local-fs.target
+"#
+    )?;
+    Ok(())
+}
+
+// misc-files/etc.mount
+pub(crate) fn gen_etc_mount(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"[Unit]
+Description=etc Dir
+DefaultDependencies=no
+Conflicts=umount.target
+Before=local-fs.target umount.target
+Wants=persist.mount
+After=persist.mount
+
+[Mount]
+What=overlay
+Where=/etc
+Type=overlay
+Options=upperdir=/persist/etc,lowerdir=/etc,workdir=/persist/etcwork
+
+[Install]
+WantedBy=local-fs.target
+"#
+    )?;
+    Ok(())
+}
+
+// misc-files/os-agent.service
+pub(crate) fn gen_os_agent_service(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"[Unit]
+Description=Agent For KubeOS
+
+[Service]
+Environment=GOTRACEBACK=crash
+ExecStart=/usr/bin/os-agent
+KillMode=process
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target
+"#
+    )?;
+    Ok(())
+}
+
+// misc-files/os-release
+pub(crate) fn gen_os_release(file: &mut File) -> Result<()> {
+    writeln!(
+        file,
+        r#"NAME=KubeOS
+ID=KubeOS
+"#
+    )?;
+    Ok(())
+}
+
+// misc-files/persist.mount
+pub(crate) fn gen_persist_mount(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"[Unit]
+Description=PERSIST Dir (/persist)
+DefaultDependencies=no
+Conflicts=umount.target
+Before=local-fs.target umount.target
+
+[Mount]
+What=/dev/disk/by-label/PERSIST
+Where=/persist
+Type=ext4
+Options=defaults
+
+[Install]
+WantedBy=local-fs.target
+"#
+    )?;
+    Ok(())
+}
+
+// misc-files/var.mount
+pub(crate) fn gen_var_mount(file: &mut File) -> Result<()> {
+    gen_copyright(file)?;
+
+    writeln!(
+        file,
+        r#"[Unit]
+Description=var Dir
+DefaultDependencies=no
+Conflicts=umount.target
+Before=local-fs.target umount.target
+Wants=persist.mount
+After=persist.mount
+
+[Mount]
+What=/persist/var
+Where=/var
+Type=node
+Options=bind
+
+[Install]
+WantedBy=local-fs.target
+"#
+    )?;
+    Ok(())
+}
+/* endregion */
diff --git a/KubeOS-Rust/kbimg/src/utils.rs b/KubeOS-Rust/kbimg/src/utils.rs
new file mode 100644
index 0000000000000000000000000000000000000000..c7ea550562d1080434e4288b9ff25fd8f2fcac26
--- /dev/null
+++ b/KubeOS-Rust/kbimg/src/utils.rs
@@ -0,0 +1,126 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{fs, os::unix::fs::PermissionsExt, path::PathBuf, process::Command};
+
+use anyhow::bail;
+
+use crate::commands::PxeConfig;
+
+pub(crate) fn execute_scripts(script: PathBuf) -> anyhow::Result<()> {
+    if !script.exists() {
+        bail!("Script does not exist: {:?}", script);
+    }
+    let status = Command::new("bash").arg(&script).status()?;
+    if !status.success() {
+        bail!("Failed to execute script: {}\n", script.display());
+    }
+    Ok(())
+}
+
+pub(crate) fn set_permissions(path: &str, permission_value: u32) -> anyhow::Result<()> {
+    let metadata = fs::metadata(path)?;
+    let mut permissions = metadata.permissions();
+    permissions.set_mode(permission_value);
+    fs::set_permissions(path, permissions)?;
+    Ok(())
+}
+
+/// Check if the input parameter is valid
+pub(crate) fn is_valid_param<S: AsRef<str> + std::fmt::Debug>(param: S) -> bool {
+    let special_chars = vec!["|", ";", "&", "&&", "||", ">", ">>", "<", ",", "#", "!", "$"];
+    !param.as_ref().chars().any(|c| special_chars.contains(&c.to_string().as_str()))
+}
+
+/// Check if the path exists and is indeed a file
+pub(crate) fn is_file_valid(msg: &str, path: &PathBuf) -> anyhow::Result<()> {
+    if !path.exists() {
+        bail!("{} does not exist: {:?}", msg, path);
+    }
+    if !path.is_file() {
+        bail!("{} exists but is not a file: {:?}", msg, path);
+    }
+    Ok(())
+}
+
+/// Check if addr is valid
+pub(crate) fn is_addr_valid(addr: &str) -> bool {
+    let ip_pattern = regex::Regex::new(r"^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$").unwrap();
+    if !ip_pattern.is_match(addr) {
+        return false;
+    }
+
+    for quad in addr.split('.') {
+        if let Ok(num) = quad.parse::<u32>() {
+            if num <= 255 {
+                continue;
+            }
+        }
+        return false;
+    }
+
+    true
+}
+
+/// Check pxe config
+pub(crate) fn check_pxe_conf_valid(pxe_config: &PxeConfig) -> anyhow::Result<()> {
+    if !is_addr_valid(&pxe_config.server_ip) {
+        bail!("address {} is invalid, please check input", &pxe_config.server_ip)
+    }
+    if !is_addr_valid(&pxe_config.local_ip) {
+        bail!("address {} is invalid, please check input", &pxe_config.local_ip)
+    }
+    if !is_addr_valid(&pxe_config.route_ip) {
+        bail!("address {} is invalid, please check input", &pxe_config.route_ip)
+    }
+    if !is_addr_valid(&pxe_config.netmask) {
+        bail!("address {} is invalid, please check input", &pxe_config.netmask)
+    }
+    Ok(())
+}
+
+/// Get architecture
+pub(crate) fn get_arch() -> String {
+    let output = std::process::Command::new("arch").output().expect("Failed to execute `arch` command");
+    String::from_utf8_lossy(&output.stdout).trim().to_string()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+
+    #[test]
+    fn test_is_valid_param() {
+        init();
+        assert_eq!(is_valid_param("test"), true);
+        assert_eq!(is_valid_param("test|test"), false);
+        assert_eq!(is_valid_param("test;test"), false);
+        assert_eq!(is_valid_param("test&test"), false);
+        assert_eq!(is_valid_param("test&&test"), false);
+        assert_eq!(is_valid_param("test||test"), false);
+        assert_eq!(is_valid_param("test>test"), false);
+        assert_eq!(is_valid_param("test>>test"), false);
+        assert_eq!(is_valid_param("test<test"), false);
+        assert_eq!(is_valid_param("test,test"), false);
+        assert_eq!(is_valid_param("test#test"), false);
+        assert_eq!(is_valid_param("test!test"), false);
+        assert_eq!(is_valid_param("test$test"), false);
+    }
+}
diff --git a/KubeOS-Rust/kbimg/src/values.rs b/KubeOS-Rust/kbimg/src/values.rs
new file mode 100644
index 0000000000000000000000000000000000000000..3d0cd95e91832fbd0740f7ba4e3256b1d8ea4e43
--- /dev/null
+++ b/KubeOS-Rust/kbimg/src/values.rs
@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+pub(crate) const SCRIPTS_DIR: &str = "./scripts-auto";
+pub(crate) const KBIMG_SH: &str = "kbimg.sh";
+pub(crate) const BOOTLOADER_SH: &str = "bootloader.sh";
+pub(crate) const SET_IN_CHROOT_SH: &str = "set_in_chroot.sh";
+pub(crate) const GRUB_CFG: &str = "grub.cfg";
+pub(crate) const RPMLIST: &str = "rpmlist";
+pub(crate) const DOCKERFILE: &str = "Dockerfile";
+
+pub(crate) const BOOTUP_DIR: &str = "./scripts-auto/00bootup";
+pub(crate) const BOOTUP_GLOBAL_CFG: &str = "Global.cfg";
+pub(crate) const BOOTUP_MODULE_SETUP_SH: &str = "module-setup.sh";
+pub(crate) const BOOTUP_MOUNT_SH: &str = "mount.sh";
+
+pub(crate) const ADMIN_CONTAINER_DIR: &str = "./scripts-auto/admin-container";
+pub(crate) const ADMIN_DOCKERFILE: &str = "Dockerfile";
+pub(crate) const ADMIN_SET_SSH_PUB_KEY_SERVICE: &str = "set-ssh-pub-key.service";
+pub(crate) const ADMIN_SET_SSH_PUB_KEY_SH: &str = "set-ssh-pub-key.sh";
+
+pub(crate) const MISC_FILES_DIR: &str = "./scripts-auto/misc-files";
+pub(crate) const MISC_BOOT_EFI_MOUNT: &str = "boot-efi.mount";
+pub(crate) const MISC_BOOT_GRUB2_MOUNT: &str = "boot-grub2.mount";
+pub(crate) const MISC_ETC_MOUNT: &str = "etc.mount";
+pub(crate) const MISC_OS_AGENT_SERVICE: &str = "os-agent.service";
+pub(crate) const MISC_OS_RELEASE: &str = "os-release";
+pub(crate) const MISC_PERSIST_MOUNT: &str = "persist.mount";
+pub(crate) const MISC_VAR_MOUNT: &str = "var.mount";
+
+// permissions
+pub(crate) const CONFIG_PERMISSION: u32 = 0o640;
+pub(crate) const EXEC_PERMISSION: u32 = 0o550;
diff --git a/KubeOS-Rust/manager/Cargo.toml b/KubeOS-Rust/manager/Cargo.toml
new file mode 100644
index 0000000000000000000000000000000000000000..f60a7c081c5b0469a8fc69a4c5359f6851eca7db
--- /dev/null
+++ b/KubeOS-Rust/manager/Cargo.toml
@@ -0,0 +1,25 @@
+[package]
+description = "KubeOS os-agent manager"
+edition = "2021"
+license = "MulanPSL-2.0"
+name = "manager"
+version = "1.0.6"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[dev-dependencies]
+mockall = { version = "=0.11.3" }
+mockito = { version = "0.31.1", default-features = false }
+predicates = { version = "=2.0.1" }
+tempfile = { version = "3.6.0" }
+
+[dependencies]
+anyhow = { version = "1.0" }
+env_logger = { version = "0.9" }
+lazy_static = { version = "1.4" }
+log = { version = "0.4" }
+nix = { version = "0.26.2" }
+regex = { version = "1.7.3" }
+reqwest = { version = "=0.12.2", features = ["blocking", "rustls-tls"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = { version = "1.0" }
+sha2 = { version = "0.10.8" }
diff --git a/KubeOS-Rust/manager/src/api/agent_status.rs b/KubeOS-Rust/manager/src/api/agent_status.rs
new file mode 100644
index 0000000000000000000000000000000000000000..bb16e6bc7ee61113eac48f9f73300ab85234aaf4
--- /dev/null
+++ b/KubeOS-Rust/manager/src/api/agent_status.rs
@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use serde::{Deserialize, Serialize};
+
+#[derive(Serialize, Deserialize, Clone, Copy, PartialEq, Eq, Debug)]
+pub enum AgentStatus {
+    UpgradeReady,
+    Upgraded,
+    Rollbacked,
+    Configured,
+}
diff --git a/KubeOS-Rust/manager/src/api/mod.rs b/KubeOS-Rust/manager/src/api/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..01c9df1a3715e0c33daf778d23c115c6d36c2fdf
--- /dev/null
+++ b/KubeOS-Rust/manager/src/api/mod.rs
@@ -0,0 +1,17 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+mod agent_status;
+mod types;
+
+pub use agent_status::*;
+pub use types::*;
diff --git a/KubeOS-Rust/manager/src/api/types.rs b/KubeOS-Rust/manager/src/api/types.rs
new file mode 100644
index 0000000000000000000000000000000000000000..98aeaa3342e94ac887248ce85c858c99bc43649b
--- /dev/null
+++ b/KubeOS-Rust/manager/src/api/types.rs
@@ -0,0 +1,140 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use super::agent_status::*;
+use crate::{
+    sys_mgmt::{CtrImageHandler, DiskImageHandler, DockerImageHandler},
+    utils::{CommandExecutor, UpgradeImageManager},
+};
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct UpgradeRequest {
+    pub version: String,
+    pub check_sum: String,
+    pub image_type: String,
+    pub container_image: String,
+    pub image_url: String,
+    pub flag_safe: bool,
+    pub mtls: bool,
+    pub certs: CertsInfo,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct CertsInfo {
+    pub ca_cert: String,
+    pub client_cert: String,
+    pub client_key: String,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct KeyInfo {
+    pub value: String,
+    pub operation: String,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct Sysconfig {
+    pub model: String,
+    pub config_path: String,
+    pub contents: HashMap<String, KeyInfo>,
+}
+
+#[derive(Deserialize, Serialize, Debug)]
+pub struct ConfigureRequest {
+    pub configs: Vec<Sysconfig>,
+}
+
+#[derive(Deserialize, Serialize, Debug, PartialEq)]
+pub struct Response {
+    pub status: AgentStatus,
+}
+
+pub enum ImageType<T: CommandExecutor> {
+    Containerd(CtrImageHandler<T>),
+    Docker(DockerImageHandler<T>),
+    Disk(DiskImageHandler<T>),
+}
+
+impl<T: CommandExecutor> ImageType<T> {
+    pub fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result<UpgradeImageManager<T>> {
+        match self {
+            ImageType::Containerd(handler) => handler.download_image(req),
+            ImageType::Docker(handler) => handler.download_image(req),
+            ImageType::Disk(handler) => handler.download_image(req),
+        }
+    }
+}
+pub trait ImageHandler<T: CommandExecutor> {
+    fn download_image(&self, req: &UpgradeRequest) -> anyhow::Result<UpgradeImageManager<T>>;
+}
+
+#[cfg(test)]
+mod tests {
+    use anyhow::Result;
+    use mockall::mock;
+
+    use super::*;
+    use crate::utils::PreparePath;
+
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    #[test]
+    fn test_download_image() {
+        let req = UpgradeRequest {
+            version: "KubeOS v2".to_string(),
+            image_type: "containerd".to_string(),
+            container_image: "kubeos-temp".to_string(),
+            check_sum: "22222".to_string(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+
+        let mut mock_executor1 = MockCommandExec::new();
+        mock_executor1.expect_run_command().returning(|_, _| Ok(()));
+        mock_executor1.expect_run_command_with_output().returning(|_, _| Ok(String::new()));
+        let c_handler = CtrImageHandler::new(PreparePath::default(), mock_executor1);
+        let image_type = ImageType::Containerd(c_handler);
+        let result = image_type.download_image(&req);
+        assert!(result.is_err());
+
+        let mut mock_executor2 = MockCommandExec::new();
+        mock_executor2.expect_run_command().returning(|_, _| Ok(()));
+        mock_executor2.expect_run_command_with_output().returning(|_, _| Ok(String::new()));
+        let docker_handler = DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor2);
+        let image_type = ImageType::Docker(docker_handler);
+        let result = image_type.download_image(&req);
+        assert!(result.is_err());
+
+        let mut mock_executor3 = MockCommandExec::new();
+        mock_executor3.expect_run_command().returning(|_, _| Ok(()));
+        mock_executor3.expect_run_command_with_output().returning(|_, _| Ok(String::new()));
+        let disk_handler = DiskImageHandler::new(PreparePath::default(), mock_executor3, "test".into());
+        let image_type = ImageType::Disk(disk_handler);
+        let result = image_type.download_image(&req);
+        assert!(result.is_err());
+    }
+}
diff --git a/KubeOS-Rust/manager/src/lib.rs b/KubeOS-Rust/manager/src/lib.rs
new file mode 100644
index 0000000000000000000000000000000000000000..b45cab999c803f2d78699e07586fb2b7d81f0d48
--- /dev/null
+++ b/KubeOS-Rust/manager/src/lib.rs
@@ -0,0 +1,15 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+pub mod api;
+pub mod sys_mgmt;
+pub mod utils;
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/config.rs b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
new file mode 100644
index 0000000000000000000000000000000000000000..138df9da24773c1ad3ca17162197c1093dc4dd60
--- /dev/null
+++ b/KubeOS-Rust/manager/src/sys_mgmt/config.rs
@@ -0,0 +1,558 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{
+    collections::HashMap,
+    fs::{self, File},
+    io::{self, BufRead, BufWriter, Write},
+    os::unix::fs::PermissionsExt,
+    path::{Path, PathBuf},
+    string::String,
+};
+
+use anyhow::{bail, Context, Result};
+use lazy_static::lazy_static;
+use log::{debug, info, trace, warn};
+use regex::Regex;
+
+use crate::{api::*, sys_mgmt::values, utils::*};
+
+lazy_static! {
+    pub static ref CONFIG_TEMPLATE: HashMap<String, Box<dyn Configuration + Sync>> = {
+        let mut config_map = HashMap::new();
+        config_map.insert(
+            values::KERNEL_SYSCTL.to_string(),
+            Box::new(KernelSysctl::new(values::DEFAULT_PROC_PATH)) as Box<dyn Configuration + Sync>,
+        );
+        config_map.insert(
+            values::KERNEL_SYSCTL_PERSIST.to_string(),
+            Box::new(KernelSysctlPersist) as Box<dyn Configuration + Sync>,
+        );
+        config_map.insert(
+            values::GRUB_CMDLINE_CURRENT.to_string(),
+            Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: true })
+                as Box<dyn Configuration + Sync>,
+        );
+        config_map.insert(
+            values::GRUB_CMDLINE_NEXT.to_string(),
+            Box::new(GrubCmdline { grub_path: values::DEFAULT_GRUB_CFG_PATH.to_string(), is_cur_partition: false })
+                as Box<dyn Configuration + Sync>,
+        );
+        config_map
+    };
+}
+
+pub trait Configuration {
+    fn set_config(&self, config: &mut Sysconfig) -> Result<()>;
+}
+
+pub struct KernelSysctl {
+    pub proc_path: String,
+}
+pub struct KernelSysctlPersist;
+pub struct GrubCmdline {
+    pub grub_path: String,
+    pub is_cur_partition: bool,
+}
+
+impl Configuration for KernelSysctl {
+    fn set_config(&self, config: &mut Sysconfig) -> Result<()> {
+        info!("Start setting kernel.sysctl");
+        for (key, key_info) in config.contents.iter() {
+            let proc_path = self.get_proc_path(key);
+            if key_info.operation == "delete" {
+                warn!("Failed to delete kernel.sysctl config with key \"{}\"", key);
+            } else if !key_info.value.is_empty() && key_info.operation.is_empty() {
+                fs::write(&proc_path, format!("{}\n", &key_info.value).as_bytes())
+                    .with_context(|| format!("Failed to write kernel.sysctl with key: \"{}\"", key))?;
+                info!("Configured kernel.sysctl {}={}", key, key_info.value);
+            } else {
+                warn!(
+                    "Failed to parse kernel.sysctl, key: \"{}\", value: \"{}\", operation: \"{}\"",
+                    key, key_info.value, key_info.operation
+                );
+            }
+        }
+        Ok(())
+    }
+}
+
+impl KernelSysctl {
+    fn new(proc_path: &str) -> Self {
+        Self { proc_path: String::from(proc_path) }
+    }
+
+    fn get_proc_path(&self, key: &str) -> PathBuf {
+        let path_str = format!("{}{}", self.proc_path, key.replace('.', "/"));
+        Path::new(&path_str).to_path_buf()
+    }
+}
+
+impl Configuration for KernelSysctlPersist {
+    fn set_config(&self, config: &mut Sysconfig) -> Result<()> {
+        info!("Start setting kernel.sysctl.persist");
+        let mut config_path = &values::DEFAULT_KERNEL_CONFIG_PATH.to_string();
+        if !config.config_path.is_empty() {
+            config_path = &config.config_path;
+        }
+        debug!("kernel.sysctl.persist config_path: \"{}\"", config_path);
+        create_config_file(config_path).with_context(|| format!("Failed to find config path \"{}\"", config_path))?;
+        let configs = get_and_set_configs(&mut config.contents, config_path)
+            .with_context(|| format!("Failed to set persist kernel configs \"{}\"", config_path))?;
+        write_configs_to_file(config_path, &configs).with_context(|| "Failed to write configs to file".to_string())?;
+        Ok(())
+    }
+}
+
+fn create_config_file(config_path: &str) -> Result<()> {
+    if !is_file_exist(config_path) {
+        let f = fs::File::create(config_path)?;
+        let metadata = f.metadata()?;
+        let mut permissions = metadata.permissions();
+        permissions.set_mode(values::DEFAULT_KERNEL_CONFIG_PERM);
+        debug!("Create file {} with permission 0644", config_path);
+    }
+    Ok(())
+}
+
+fn get_and_set_configs(expect_configs: &mut HashMap<String, KeyInfo>, config_path: &str) -> Result<Vec<String>> {
+    let f = File::open(config_path).with_context(|| format!("Failed to open config path \"{}\"", config_path))?;
+    let mut configs_write = Vec::new();
+    for line in io::BufReader::new(f).lines() {
+        let line = line?;
+        // if line is a comment or blank
+        if line.starts_with('#') || line.starts_with(';') || line.trim().is_empty() {
+            configs_write.push(line);
+            continue;
+        }
+        let config_kv: Vec<&str> = line.splitn(2, '=').map(|s| s.trim()).collect();
+        // if config_kv is not a key-value pair
+        if config_kv.len() != 2 {
+            bail!("could not parse sysctl config {}", line);
+        }
+        let new_key_info = expect_configs.get(config_kv[0]);
+        let new_config = match new_key_info {
+            Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config_kv, new_key_info),
+            Some(new_key_info) => handle_update_key(&config_kv, new_key_info),
+            None => config_kv.join("="),
+        };
+        configs_write.push(new_config);
+        expect_configs.remove(config_kv[0]);
+    }
+    let new_config = handle_add_key(expect_configs, false);
+    configs_write.extend(new_config);
+    Ok(configs_write)
+}
+
+fn write_configs_to_file(config_path: &str, configs: &Vec<String>) -> Result<()> {
+    info!("Write configuration to file \"{}\"", config_path);
+    let f = File::create(config_path)?;
+    let mut w = BufWriter::new(f);
+    for line in configs {
+        if line.is_empty() {
+            continue;
+        }
+        writeln!(w, "{}", line.as_str())?;
+    }
+    w.flush().with_context(|| format!("Failed to flush file {}", config_path))?;
+    w.get_mut().sync_all().with_context(|| "Failed to sync".to_string())?;
+    debug!("Write configuration to file \"{}\" success", config_path);
+    Ok(())
+}
+
+fn handle_delete_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String {
+    let key = config_kv[0];
+    if config_kv.len() == 1 && new_config_info.value.is_empty() {
+        info!("Delete configuration key: \"{}\"", key);
+        return String::from("");
+    } else if config_kv.len() == 1 && !new_config_info.value.is_empty() {
+        warn!("Failed to delete key \"{}\" with inconsistent values \"nil\" and \"{}\"", key, new_config_info.value);
+        return key.to_string();
+    }
+    let old_value = config_kv[1];
+    if old_value != new_config_info.value {
+        warn!(
+            "Failed to delete key \"{}\" with inconsistent values \"{}\" and \"{}\"",
+            key, old_value, new_config_info.value
+        );
+        return config_kv.join("=");
+    }
+    info!("Delete configuration {}={}", key, old_value);
+    String::new()
+}
+
+fn handle_update_key(config_kv: &[&str], new_config_info: &KeyInfo) -> String {
+    let key = config_kv[0];
+    if !new_config_info.operation.is_empty() {
+        warn!(
+            "Unknown operation \"{}\", updating key \"{}\" with value \"{}\" by default",
+            new_config_info.operation, key, new_config_info.value
+        );
+    }
+    if config_kv.len() == values::ONLY_KEY && new_config_info.value.is_empty() {
+        return key.to_string();
+    }
+    let new_value = new_config_info.value.trim();
+    if config_kv.len() == values::ONLY_KEY && !new_config_info.value.is_empty() {
+        info!("Update configuration \"{}={}\"", key, new_value);
+        return format!("{}={}", key, new_value);
+    }
+    if new_config_info.value.is_empty() {
+        warn!("Failed to update key \"{}\" with \"null\" value", key);
+        return config_kv.join("=");
+    }
+    info!("Update configuration \"{}={}\"", key, new_value);
+    format!("{}={}", key, new_value)
+}
+
+fn handle_add_key(expect_configs: &HashMap<String, KeyInfo>, is_only_key_valid: bool) -> Vec<String> {
+    let mut configs_write = Vec::new();
+    for (key, config_info) in expect_configs.iter() {
+        if config_info.operation == "delete" {
+            warn!("Failed to delete inexistent key: \"{}\"", key);
+            continue;
+        }
+        if key.is_empty() || key.contains('=') {
+            warn!("Failed to add \"null\" key or key containing \"=\", key: \"{}\"", key);
+            continue;
+        }
+        if !config_info.operation.is_empty() {
+            warn!(
+                "Unknown operation \"{}\", adding key \"{}\" with value \"{}\" by default",
+                config_info.operation, key, config_info.value
+            );
+        }
+        let (k, v) = (key.trim(), config_info.value.trim());
+        if v.is_empty() && is_only_key_valid {
+            info!("Add configuration \"{}\"", k);
+            configs_write.push(k.to_string());
+        } else if v.is_empty() {
+            warn!("Failed to add key \"{}\" with \"null\" value", k);
+        } else {
+            info!("Add configuration \"{}={}\"", k, v);
+            configs_write.push(format!("{}={}", k, v));
+        }
+    }
+    configs_write
+}
+
+impl Configuration for GrubCmdline {
+    fn set_config(&self, config: &mut Sysconfig) -> Result<()> {
+        if self.is_cur_partition {
+            info!("Start setting grub.cmdline.current configuration");
+        } else {
+            info!("Start setting grub.cmdline.next configuration");
+        }
+        if !is_file_exist(&self.grub_path) {
+            bail!("Failed to find grub.cfg file");
+        }
+        let config_partition = if cfg!(test) {
+            self.is_cur_partition
+        } else {
+            self.get_config_partition(RealCommandExecutor {})
+                .with_context(|| "Failed to get config partition".to_string())?
+        };
+        debug!("Config_partition: {} (false means partition A, true means partition B)", config_partition);
+        let configs = get_and_set_grubcfg(&mut config.contents, &self.grub_path, config_partition)
+            .with_context(|| "Failed to set grub configs".to_string())?;
+        write_configs_to_file(&self.grub_path, &configs)
+            .with_context(|| "Failed to write configs to file".to_string())?;
+        Ok(())
+    }
+}
+
+impl GrubCmdline {
+    // get_config_partition returns false if the menuentry to be configured is A, true for menuentry B
+    fn get_config_partition<T: CommandExecutor>(&self, executor: T) -> Result<bool> {
+        let (_, next_partition) = get_partition_info(&executor)?;
+        let mut flag = false;
+        if next_partition.menuentry == "B" {
+            flag = true
+        }
+        Ok(self.is_cur_partition != flag)
+    }
+}
+
+fn get_and_set_grubcfg(
+    expect_configs: &mut HashMap<String, KeyInfo>,
+    grub_path: &str,
+    config_partition: bool,
+) -> Result<Vec<String>> {
+    let f = File::open(grub_path).with_context(|| format!("Failed to open grub.cfg \"{}\"", grub_path))?;
+    let re_find_cur_linux = r"^\s*linux.*root=.*";
+    let re = Regex::new(re_find_cur_linux)?;
+    let mut configs_write = Vec::new();
+    let mut match_config_partition = false;
+    for line in io::BufReader::new(f).lines() {
+        let mut line = line?;
+        if re.is_match(&line) {
+            if match_config_partition == config_partition {
+                line = modify_boot_cfg(expect_configs, &line)?;
+            }
+            match_config_partition = true;
+        }
+        configs_write.push(line);
+    }
+    Ok(configs_write)
+}
+
+fn modify_boot_cfg(expect_configs: &mut HashMap<String, KeyInfo>, line: &String) -> Result<String> {
+    trace!("Match partition that need to be configured, entering modify_boot_cfg, linux line: {}", line);
+    let mut new_configs = vec!["       ".to_string()];
+    let olg_configs: Vec<&str> = line.split(' ').collect();
+    for old_config in olg_configs {
+        if old_config.is_empty() {
+            continue;
+        }
+        // At most 2 substrings can be returned to satisfy the case like root=UUID=xxxx
+        let config = old_config.splitn(2, '=').collect::<Vec<&str>>();
+        if config.len() != values::ONLY_KEY && config.len() != values::KV_PAIR {
+            bail!("Failed to parse grub.cfg linux line {}", old_config);
+        }
+        let new_key_info = expect_configs.get(config[0]);
+        let new_config = match new_key_info {
+            Some(new_key_info) if new_key_info.operation == "delete" => handle_delete_key(&config, new_key_info),
+            Some(new_key_info) => handle_update_key(&config, new_key_info),
+            None => config.join("="),
+        };
+        if !new_config.is_empty() {
+            new_configs.push(new_config);
+        }
+        expect_configs.remove(config[0]);
+    }
+    let new_config = handle_add_key(expect_configs, true);
+    new_configs.extend(new_config);
+    Ok(new_configs.join(" "))
+}
+
+#[cfg(test)]
+mod tests {
+    use std::fs;
+
+    use mockall::{mock, predicate::*};
+    use tempfile::{NamedTempFile, TempDir};
+
+    use super::*;
+    use crate::sys_mgmt::{GRUB_CMDLINE_CURRENT, GRUB_CMDLINE_NEXT, KERNEL_SYSCTL, KERNEL_SYSCTL_PERSIST};
+
+    // Mock the CommandExecutor trait
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+
+    #[test]
+    fn test_get_config_partition() {
+        init();
+        let mut grub_cmdline = GrubCmdline { grub_path: String::from(""), is_cur_partition: true };
+        let mut executor = MockCommandExec::new();
+
+        // the output shows that current root menuentry is A
+        let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3  ext4\nsda4 /persist ext4\nsr0  iso9660\n";
+        executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string()));
+
+        let result = grub_cmdline.get_config_partition(executor).unwrap();
+        // it should return false because the current root menuentry is A and we want to configure current partition
+        assert_eq!(result, false);
+
+        let mut executor = MockCommandExec::new();
+
+        // the output shows that current root menuentry is A
+        let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3  ext4\nsda4 /persist ext4\nsr0  iso9660\n";
+        executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string()));
+        grub_cmdline.is_cur_partition = false;
+        let result = grub_cmdline.get_config_partition(executor).unwrap();
+        // it should return true because the current root menuentry is A and we want to configure next partition
+        assert_eq!(result, true);
+    }
+
+    #[test]
+    fn test_kernel_sysctl() {
+        init();
+        let tmp_dir = TempDir::new().unwrap();
+        assert_eq!(tmp_dir.path().exists(), true);
+        let kernel_sysctl = KernelSysctl::new(tmp_dir.path().to_str().unwrap());
+
+        let config_detail = HashMap::from([
+            ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }),
+            ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }),
+            ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }),
+            ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }),
+            ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }),
+        ]);
+
+        let mut config =
+            Sysconfig { model: KERNEL_SYSCTL.to_string(), config_path: String::from(""), contents: config_detail };
+        kernel_sysctl.set_config(&mut config).unwrap();
+
+        let result = fs::read_to_string(format!("{}{}", tmp_dir.path().to_str().unwrap(), "a")).unwrap();
+        assert_eq!(result, "1\n");
+    }
+
+    #[test]
+    fn test_kernel_sysctl_persist() {
+        init();
+        let comment = r"# This file is managed by KubeOS for unit testing.";
+        // create a tmp file with comment
+        let mut tmp_file = tempfile::NamedTempFile::new().unwrap();
+        writeln!(tmp_file, "{}", comment).unwrap();
+        writeln!(tmp_file, "a=0").unwrap();
+        writeln!(tmp_file, "d=4").unwrap();
+        writeln!(tmp_file, "e=5").unwrap();
+        writeln!(tmp_file, "g=7").unwrap();
+        let kernel_sysctl_persist = KernelSysctlPersist {};
+        let config_detail = HashMap::from([
+            ("a".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }),
+            ("b".to_string(), KeyInfo { value: "2".to_string(), operation: "delete".to_string() }),
+            ("c".to_string(), KeyInfo { value: "3".to_string(), operation: "add".to_string() }),
+            ("d".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }),
+            ("e".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }),
+            ("f".to_string(), KeyInfo { value: "".to_string(), operation: "add".to_string() }),
+            ("g".to_string(), KeyInfo { value: "7".to_string(), operation: "delete".to_string() }),
+            ("".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }),
+            ("s=x".to_string(), KeyInfo { value: "8".to_string(), operation: "".to_string() }),
+        ]);
+        let mut config = Sysconfig {
+            model: KERNEL_SYSCTL_PERSIST.to_string(),
+            config_path: String::from(tmp_file.path().to_str().unwrap()),
+            contents: config_detail,
+        };
+        kernel_sysctl_persist.set_config(&mut config).unwrap();
+        let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap();
+        let expected_res = format!("{}\n{}\n{}\n{}\n{}\n", comment, "a=1", "d=4", "e=5", "c=3");
+        assert_eq!(result, expected_res);
+        let mut config = Sysconfig {
+            model: KERNEL_SYSCTL_PERSIST.to_string(),
+            config_path: String::from("/tmp/kubeos-test-kernel-sysctl-persist.txt"),
+            contents: HashMap::new(),
+        };
+        kernel_sysctl_persist.set_config(&mut config).unwrap();
+        assert!(is_file_exist(&config.config_path));
+        delete_file_or_dir(&config.config_path).unwrap();
+    }
+
+    #[test]
+    fn write_configs_to_file_tests() {
+        init();
+        let tmp_file = NamedTempFile::new().unwrap();
+        let configs = vec!["a=1".to_string(), "b=2".to_string()];
+        write_configs_to_file(tmp_file.path().to_str().unwrap(), &configs).unwrap();
+        assert_eq!(fs::read(tmp_file.path()).unwrap(), b"a=1\nb=2\n");
+    }
+
+    #[test]
+    fn test_grub_cmdline() {
+        init();
+        let mut tmp_file = NamedTempFile::new().unwrap();
+        let mut grub_cmdline =
+            GrubCmdline { grub_path: tmp_file.path().to_str().unwrap().to_string(), is_cur_partition: true };
+        let grub_cfg = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' {
+        load_video
+        set gfxpayload=keep
+        insmod gzio
+        insmod part_gpt
+        insmod ext2
+        set root='hd0,gpt2'
+        linux   /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3
+        initrd  /boot/initramfs.img
+}
+
+menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' {
+        load_video
+        set gfxpayload=keep
+        insmod gzio
+        insmod part_gpt
+        insmod ext2
+        set root='hd0,gpt3'
+        linux   /boot/vmlinuz root=UUID=2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3
+        initrd  /boot/initramfs.img
+}";
+        writeln!(tmp_file, "{}", grub_cfg).unwrap();
+        let config_second_part = HashMap::from([
+            ("debug".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }),
+            ("quiet".to_string(), KeyInfo { value: "".to_string(), operation: "delete".to_string() }),
+            ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "".to_string() }),
+            ("nomodeset".to_string(), KeyInfo { value: "".to_string(), operation: "update".to_string() }),
+            ("oops".to_string(), KeyInfo { value: "".to_string(), operation: "".to_string() }),
+            ("".to_string(), KeyInfo { value: "test".to_string(), operation: "".to_string() }),
+            ("selinux".to_string(), KeyInfo { value: "1".to_string(), operation: "delete".to_string() }),
+            ("acpi".to_string(), KeyInfo { value: "off".to_string(), operation: "delete".to_string() }),
+            ("ro".to_string(), KeyInfo { value: "1".to_string(), operation: "".to_string() }),
+        ]);
+        let mut config = Sysconfig {
+            model: GRUB_CMDLINE_CURRENT.to_string(),
+            config_path: String::new(),
+            contents: config_second_part,
+        };
+        grub_cmdline.set_config(&mut config).unwrap();
+        grub_cmdline.is_cur_partition = false;
+        let config_first_part = HashMap::from([
+            ("pci".to_string(), KeyInfo { value: "nomis".to_string(), operation: "".to_string() }),
+            ("quiet".to_string(), KeyInfo { value: "11".to_string(), operation: "delete".to_string() }),
+            ("panic".to_string(), KeyInfo { value: "5".to_string(), operation: "update".to_string() }),
+        ]);
+        config.contents = config_first_part;
+        config.model = GRUB_CMDLINE_NEXT.to_string();
+        grub_cmdline.set_config(&mut config).unwrap();
+        let result = fs::read_to_string(tmp_file.path().to_str().unwrap()).unwrap();
+        let expected_res = r"menuentry 'A' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-A' {
+        load_video
+        set gfxpayload=keep
+        insmod gzio
+        insmod part_gpt
+        insmod ext2
+        set root='hd0,gpt2'
+        linux /boot/vmlinuz root=UUID=1 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 pci=nomis
+        initrd  /boot/initramfs.img
+}
+menuentry 'B' --class KubeOS --class gnu-linux --class gnu --class os --unrestricted $menuentry_id_option 'KubeOS-B' {
+        load_video
+        set gfxpayload=keep
+        insmod gzio
+        insmod part_gpt
+        insmod ext2
+        set root='hd0,gpt3'
+        linux /boot/vmlinuz root=UUID=2 ro=1 rootfstype=ext4 nomodeset oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=5 debug
+        initrd  /boot/initramfs.img
+}
+";
+        assert_eq!(result, expected_res);
+
+        // test grub.cfg not exist
+        grub_cmdline.grub_path = "/tmp/grub-KubeOS-test.cfg".to_string();
+        let res = grub_cmdline.set_config(&mut config);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_create_config_file() {
+        init();
+        let tmp_file = "/tmp/kubeos-test-create-config-file.txt";
+        create_config_file(&tmp_file).unwrap();
+        assert!(is_file_exist(&tmp_file));
+        fs::remove_file(tmp_file).unwrap();
+    }
+}
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
new file mode 100644
index 0000000000000000000000000000000000000000..80caf291b9c0c0001293abb71047f8c516d91876
--- /dev/null
+++ b/KubeOS-Rust/manager/src/sys_mgmt/containerd_image.rs
@@ -0,0 +1,301 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{fs, os::unix::fs::PermissionsExt, path::Path};
+
+use anyhow::{anyhow, Context, Result};
+use log::{debug, info};
+
+use crate::{
+    api::{ImageHandler, UpgradeRequest},
+    sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES},
+    utils::*,
+};
+
+pub struct CtrImageHandler<T: CommandExecutor> {
+    pub paths: PreparePath,
+    pub executor: T,
+}
+
+const DEFAULT_NAMESPACE: &str = "k8s.io";
+
+impl<T: CommandExecutor> ImageHandler<T> for CtrImageHandler<T> {
+    fn download_image(&self, req: &UpgradeRequest) -> Result<UpgradeImageManager<T>> {
+        perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?;
+        self.get_image(req)?;
+        self.get_rootfs_archive(req, IMAGE_PERMISSION)?;
+
+        let (_, next_partition_info) = get_partition_info(&self.executor)?;
+        let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone());
+        img_manager.create_os_image(IMAGE_PERMISSION)
+    }
+}
+
+impl Default for CtrImageHandler<RealCommandExecutor> {
+    fn default() -> Self {
+        Self { paths: PreparePath::default(), executor: RealCommandExecutor {} }
+    }
+}
+
+impl<T: CommandExecutor> CtrImageHandler<T> {
+    #[cfg(test)]
+    pub fn new(paths: PreparePath, executor: T) -> Self {
+        Self { paths, executor }
+    }
+
+    fn get_image(&self, req: &UpgradeRequest) -> Result<()> {
+        let image_name = &req.container_image;
+        is_valid_image_name(image_name)?;
+        let cli: String =
+            if is_command_available("crictl", &self.executor) { "crictl".to_string() } else { "ctr".to_string() };
+        remove_image_if_exist(&cli, image_name, &self.executor)?;
+        info!("Start pulling image {}", image_name);
+        pull_image(&cli, image_name, &self.executor)?;
+        info!("Start checking image digest");
+        check_oci_image_digest(&cli, image_name, &req.check_sum, &self.executor)?;
+        Ok(())
+    }
+
+    fn get_rootfs_archive(&self, req: &UpgradeRequest, permission: u32) -> Result<()> {
+        let image_name = &req.container_image;
+        let mount_path = &self
+            .paths
+            .mount_path
+            .to_str()
+            .ok_or_else(|| anyhow!("Failed to get mount path: {}", self.paths.mount_path.display()))?;
+        info!("Start getting rootfs {}", image_name);
+        self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?;
+        self.executor
+            .run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "mount", "--rw", image_name, mount_path])?;
+        // copy os.tar from mount_path to its partent dir
+        self.copy_file(self.paths.mount_path.join(&self.paths.rootfs_file), &self.paths.tar_path, permission)?;
+        self.check_and_unmount(mount_path).with_context(|| "Failed to clean containerd environment".to_string())?;
+        Ok(())
+    }
+
+    fn check_and_unmount(&self, mount_path: &str) -> Result<()> {
+        let ctr_snapshot_cmd =
+            format!("ctr -n={} snapshots ls | grep {} | awk '{{print $1}}'", DEFAULT_NAMESPACE, mount_path);
+        let exist_snapshot = self.executor.run_command_with_output("bash", &["-c", &ctr_snapshot_cmd])?;
+        if !exist_snapshot.is_empty() {
+            self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "images", "unmount", mount_path])?;
+            self.executor.run_command("ctr", &["-n", DEFAULT_NAMESPACE, "snapshots", "remove", mount_path])?;
+        }
+        Ok(())
+    }
+
+    fn copy_file<P: AsRef<Path>, Q: AsRef<Path>>(&self, src: P, dst: Q, permission: u32) -> Result<()> {
+        let copied_bytes = fs::copy(src.as_ref(), dst.as_ref())?;
+        debug!("Copy {} to {}, total bytes: {}", src.as_ref().display(), dst.as_ref().display(), copied_bytes);
+        fs::set_permissions(dst, fs::Permissions::from_mode(permission))?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{io::Write, path::PathBuf};
+
+    use mockall::mock;
+    use tempfile::NamedTempFile;
+
+    use super::*;
+    use crate::api::CertsInfo;
+
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+
+    #[test]
+    fn test_get_image() {
+        init();
+        let mut mock_executor = MockCommandExec::new();
+        let image_name = "docker.io/library/busybox:latest";
+        let req = UpgradeRequest {
+            version: "KubeOS v2".to_string(),
+            image_type: "containerd".to_string(),
+            container_image: image_name.to_string(),
+            check_sum: "22222".to_string(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        // mock is_command_available
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "/bin/sh" && args.contains(&"command -v crictl")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+        // mock remove_image_if_exist
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "crictl" && args.contains(&"inspecti")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "crictl" && args.contains(&"rmi")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+        // mock pull_image
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| {
+                cmd == "crictl" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest")
+            })
+            .times(1)
+            .returning(|_, _| Ok(()));
+        // mock get_oci_image_digest
+        let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]";
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| {
+                cmd == "crictl" && args.contains(&"inspecti") && args.contains(&"{{.status.repoDigests}}")
+            })
+            .times(1)
+            .returning(|_, _| Ok(command_output2.to_string()));
+        let ctr = CtrImageHandler::new(PreparePath::default(), mock_executor);
+        let result = ctr.get_image(&req);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_get_rootfs_archive() {
+        init();
+        let mut mock_executor = MockCommandExec::new();
+        let image_name = "docker.io/library/busybox:latest";
+        let req = UpgradeRequest {
+            version: "KubeOS v2".to_string(),
+            image_type: "containerd".to_string(),
+            container_image: image_name.to_string(),
+            check_sum: "22222".to_string(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+
+        // mock check_and_unmount
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok("".to_string()));
+
+        // mock ctr mount rw
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[4] == "--rw") // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+
+        // create temp file for copy
+        let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file.");
+        writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file.");
+
+        // Get the path of the temporary file and the path where it should be copied.
+        let src_dir = tmp_file.path().parent().unwrap();
+        let src_file_name = tmp_file.path().file_name().unwrap().to_str().unwrap().to_string();
+        let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file.");
+        let dst_path = dst_file.path().to_path_buf();
+
+        let paths = PreparePath {
+            persist_path: "/tmp".into(),
+            update_path: PathBuf::new(),
+            image_path: PathBuf::new(),
+            mount_path: src_dir.to_path_buf(),
+            rootfs_file: src_file_name.clone(),
+            tar_path: dst_path.clone(),
+        };
+
+        // mock check_and_unmount
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c") // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok("".to_string()));
+
+        let ctr = CtrImageHandler::new(paths, mock_executor);
+        let result = ctr.get_rootfs_archive(&req, IMAGE_PERMISSION);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_copy_file() {
+        // Setup: Create a temporary file and write some data to it.
+        let mut tmp_file = NamedTempFile::new().expect("Failed to create temporary file.");
+        writeln!(tmp_file, "Hello, world!").expect("Failed to write to temporary file.");
+
+        // Get the path of the temporary file and the path where it should be copied.
+        let src_path = tmp_file.path().to_str().unwrap().to_string();
+        let dst_file = NamedTempFile::new().expect("Failed to create destination temporary file.");
+        let dst_path = dst_file.path().to_str().unwrap().to_string();
+
+        let ctr = CtrImageHandler::default();
+        let result = ctr.copy_file(&src_path, &dst_path, IMAGE_PERMISSION);
+
+        assert!(result.is_ok());
+
+        let expected_content = "Hello, world!\n";
+        let actual_content = fs::read_to_string(&dst_path).expect("Failed to read destination file.");
+        assert_eq!(expected_content, actual_content);
+
+        // Assert the file permission
+        let metadata = fs::metadata(&dst_path).expect("Failed to read destination file.");
+        let expected_permission = 0o100600;
+        assert_eq!(metadata.permissions().mode(), expected_permission);
+    }
+
+    #[test]
+    fn test_check_and_unmount() {
+        let mut mock_executor = MockCommandExec::new();
+
+        // When `run_command_with_output` is called with "bash" and the specific args, it will return Ok("snapshot_exists").
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| cmd == "bash" && args.len() == 2 && args[0] == "-c")
+            .times(1)
+            .returning(|_, _| Ok("snapshot_exists".to_string()));
+
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "ctr" && args.contains(&"images"))
+            .times(1)
+            .returning(|_, _| Ok(()));
+
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "ctr" && args.contains(&"snapshots"))
+            .times(1)
+            .returning(|_, _| Ok(()));
+
+        let result = CtrImageHandler::new(PreparePath::default(), mock_executor).check_and_unmount("test_mount_path");
+
+        assert!(result.is_ok());
+    }
+}
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
new file mode 100644
index 0000000000000000000000000000000000000000..6d836dc4111674e7b5044661f503d4a92cc8aac5
--- /dev/null
+++ b/KubeOS-Rust/manager/src/sys_mgmt/disk_image.rs
@@ -0,0 +1,406 @@
+use std::{
+    fs,
+    os::unix::fs::PermissionsExt,
+    path::{Path, PathBuf},
+};
+
+use anyhow::{bail, Context, Result};
+use log::{debug, info, trace};
+use reqwest::{blocking::Client, Certificate};
+use sha2::{Digest, Sha256};
+
+use crate::{
+    api::{CertsInfo, ImageHandler, UpgradeRequest},
+    sys_mgmt::{CERTS_PATH, IMAGE_PERMISSION, PERSIST_DIR},
+    utils::*,
+};
+
+const BUFFER: u64 = 1024 * 1024 * 10;
+
+pub struct DiskImageHandler<T: CommandExecutor> {
+    pub paths: PreparePath,
+    pub executor: T,
+    pub certs_path: String,
+}
+
+impl<T: CommandExecutor> ImageHandler<T> for DiskImageHandler<T> {
+    fn download_image(&self, req: &UpgradeRequest) -> Result<UpgradeImageManager<T>> {
+        self.download(req)?;
+        self.checksum_match(self.paths.image_path.to_str().unwrap_or_default(), &req.check_sum)?;
+        let (_, next_partition_info) = get_partition_info(&self.executor)?;
+        let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone());
+        Ok(img_manager)
+    }
+}
+
+impl Default for DiskImageHandler<RealCommandExecutor> {
+    fn default() -> Self {
+        Self { paths: PreparePath::default(), executor: RealCommandExecutor {}, certs_path: CERTS_PATH.to_string() }
+    }
+}
+
+impl<T: CommandExecutor> DiskImageHandler<T> {
+    #[cfg(test)]
+    pub fn new(paths: PreparePath, executor: T, certs_path: String) -> Self {
+        Self { paths, executor, certs_path }
+    }
+
+    fn download(&self, req: &UpgradeRequest) -> Result<()> {
+        let mut resp = self.send_download_request(req)?;
+        if resp.status() != reqwest::StatusCode::OK {
+            bail!("Failed to download image from {}, status: {}", req.image_url, resp.status());
+        }
+        debug!("Received response body size: {:?}", resp.content_length().unwrap_or_default());
+        let need_bytes = resp.content_length().unwrap_or_default() + BUFFER;
+
+        check_disk_size(
+            i64::try_from(need_bytes).with_context(|| "Failed to transform content length from u64 to i64")?,
+            self.paths.image_path.parent().unwrap_or_else(|| Path::new(PERSIST_DIR)),
+        )?;
+
+        let mut out = fs::File::create(&self.paths.image_path)?;
+        trace!("Start to save upgrade image to path {}", &self.paths.image_path.display());
+        out.set_permissions(fs::Permissions::from_mode(IMAGE_PERMISSION))?;
+        let bytes = resp.copy_to(&mut out)?;
+        info!(
+            "Download image successfully, upgrade image path: {}, write bytes: {}",
+            &self.paths.image_path.display(),
+            bytes
+        );
+        Ok(())
+    }
+
+    fn checksum_match(&self, file_path: &str, check_sum: &str) -> Result<()> {
+        info!("Start checking image checksum");
+        let check_sum = check_sum.to_ascii_lowercase();
+        let file = fs::read(file_path)?;
+        let mut hasher = Sha256::new();
+        hasher.update(file);
+        let hash = hasher.finalize();
+        // sha256sum -b /persist/update.img
+        let cal_sum = format!("{:X}", hash).to_ascii_lowercase();
+        if cal_sum != check_sum {
+            delete_file_or_dir(file_path)?;
+            bail!("Checksum {} mismatch to {}", cal_sum, check_sum);
+        }
+        debug!("Checksum match");
+        Ok(())
+    }
+
+    fn send_download_request(&self, req: &UpgradeRequest) -> Result<reqwest::blocking::Response> {
+        let client: Client;
+
+        if !req.image_url.starts_with("https://") {
+            // http request
+            if !req.flag_safe {
+                bail!("The upgrade image url is not safe");
+            }
+            info!("Discover http request to: {}", &req.image_url);
+            client = Client::new();
+        } else if req.mtls {
+            // https mtls request
+            client = self.load_ca_client_certs(&req.certs).with_context(|| "Failed to load client certificates")?;
+            info!("Discover https mtls request to: {}", &req.image_url);
+        } else {
+            // https request
+            client = self.load_ca_certs(&req.certs.ca_cert).with_context(|| "Failed to load CA certificates")?;
+            info!("Discover https request to: {}", &req.image_url);
+        }
+
+        client.get(&req.image_url).send().with_context(|| format!("Failed to fetch from URL: {}", &req.image_url))
+    }
+
+    fn load_ca_certs(&self, ca_cert: &str) -> Result<Client> {
+        trace!("Start to load CA certificates");
+        self.cert_exist(ca_cert)?;
+        let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(ca_cert))?)?;
+        let client = Client::builder().add_root_certificate(ca).build()?;
+        Ok(client)
+    }
+
+    fn load_ca_client_certs(&self, certs: &CertsInfo) -> Result<Client> {
+        trace!("Start to load CA and client certificates");
+        self.cert_exist(&certs.ca_cert)?;
+        let ca = Certificate::from_pem(&std::fs::read(self.get_certs_path(&certs.ca_cert))?)?;
+
+        self.cert_exist(&certs.client_cert)?;
+        self.cert_exist(&certs.client_key)?;
+        let client_cert = std::fs::read(self.get_certs_path(&certs.client_cert))?;
+        let client_key = std::fs::read(self.get_certs_path(&certs.client_key))?;
+        let mut client_identity = Vec::new();
+        client_identity.extend_from_slice(&client_cert);
+        client_identity.extend_from_slice(&client_key);
+        let client_id = reqwest::Identity::from_pem(&client_identity)?;
+
+        let client = Client::builder().use_rustls_tls().add_root_certificate(ca).identity(client_id).build()?;
+        Ok(client)
+    }
+
+    fn cert_exist(&self, cert_file: &str) -> Result<()> {
+        if cert_file.is_empty() {
+            bail!("Please provide the certificate");
+        }
+        if !self.get_certs_path(cert_file).exists() {
+            bail!("Certificate does not exist: {}", cert_file);
+        }
+        Ok(())
+    }
+
+    fn get_certs_path(&self, cert: &str) -> PathBuf {
+        let cert_path = format!("{}{}", self.certs_path, cert);
+        PathBuf::from(cert_path)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::io::Write;
+
+    use mockall::mock;
+    use mockito;
+    use tempfile::NamedTempFile;
+
+    use super::*;
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    #[test]
+    fn test_get_certs_path() {
+        init();
+        let handler = DiskImageHandler::<RealCommandExecutor>::default();
+        let certs_path = handler.get_certs_path("ca.pem");
+        assert_eq!(certs_path.to_str().unwrap(), "/etc/KubeOS/certs/ca.pem");
+    }
+
+    #[test]
+    fn test_cert_exist() {
+        init();
+        // generate tmp file
+        let tmp_file = NamedTempFile::new().unwrap();
+        let handler =
+            DiskImageHandler::<RealCommandExecutor>::new(PreparePath::default(), RealCommandExecutor {}, String::new());
+        let res = handler.cert_exist(tmp_file.path().to_str().unwrap());
+        assert!(res.is_ok());
+
+        assert!(handler.cert_exist("aaa.pem").is_err());
+        assert!(handler.cert_exist("").is_err())
+    }
+
+    #[test]
+    fn test_send_download_request() {
+        init();
+        // This is a tmp cert only for KubeOS unit testing.
+        let tmp_cert = "-----BEGIN CERTIFICATE-----\n\
+        MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\
+        MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\
+        MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\
+        bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\
+        Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\
+        HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\
+        c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\
+        vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\
+        -----END CERTIFICATE-----\n";
+
+        // This is a tmp private key only for KubeOS unit testing.
+        let tmp_key = "-----BEGIN PRIVATE KEY-----\n\
+        MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\
+        Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\
+        nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\
+        -----END PRIVATE KEY-----\n";
+
+        // Create a temporary file to hold the certificate
+        let mut cert_file = NamedTempFile::new().unwrap();
+        cert_file.write_all(tmp_cert.as_bytes()).unwrap();
+        println!("cert_file: {:?}", cert_file.path().to_str().unwrap());
+
+        // Create a temporary file to hold the private key
+        let mut key_file = NamedTempFile::new().unwrap();
+        key_file.write_all(tmp_key.as_bytes()).unwrap();
+        // http
+        let handler = DiskImageHandler::<RealCommandExecutor>::default();
+        let mut req = UpgradeRequest {
+            version: "v2".into(),
+            check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(),
+            image_type: "disk".into(),
+            container_image: "".into(),
+            image_url: "http://localhost:8080/aaa.txt".to_string(),
+            flag_safe: true,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        let res = handler.send_download_request(&req);
+        assert!(res.is_err());
+        req.flag_safe = false;
+        let res = handler.send_download_request(&req);
+        assert!(res.is_err());
+
+        // https
+        let mut handler = DiskImageHandler::<RealCommandExecutor>::default();
+        handler.certs_path = "/tmp".to_string();
+        let tmp_cert_filename = cert_file.path().file_name().unwrap().to_str().unwrap();
+        let req = UpgradeRequest {
+            version: "v2".into(),
+            check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(),
+            image_type: "disk".into(),
+            container_image: "".into(),
+            image_url: "https://localhost:8081/aaa.txt".to_string(),
+            flag_safe: true,
+            mtls: false,
+            certs: CertsInfo {
+                ca_cert: tmp_cert_filename.to_string(),
+                client_cert: "".to_string(),
+                client_key: "".to_string(),
+            },
+        };
+        let res = handler.send_download_request(&req);
+        assert!(res.is_err());
+
+        // mtls
+        let tmp_key = NamedTempFile::new().unwrap();
+        let tmp_key_filename = tmp_key.path().file_name().unwrap().to_str().unwrap();
+        let mut handler = DiskImageHandler::<RealCommandExecutor>::default();
+        handler.certs_path = "/tmp".to_string();
+        let req = UpgradeRequest {
+            version: "v2".into(),
+            check_sum: "1327e27d600538354d93bd68cce86566dd089e240c126dc3019cafabdc65aa02".into(),
+            image_type: "disk".into(),
+            container_image: "".into(),
+            image_url: "https://localhost:8082/aaa.txt".to_string(),
+            flag_safe: true,
+            mtls: true,
+            certs: CertsInfo {
+                ca_cert: tmp_cert_filename.to_string(),
+                client_cert: tmp_cert_filename.to_string(),
+                client_key: tmp_key_filename.to_string(),
+            },
+        };
+        let res = handler.send_download_request(&req);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_checksum_match() {
+        init();
+        let mut tmp_file = NamedTempFile::new().unwrap();
+        tmp_file.write(b"This is a test txt file for KubeOS test.\n").unwrap();
+        let mut handler = DiskImageHandler::<RealCommandExecutor>::default();
+        handler.paths.image_path = tmp_file.path().to_path_buf();
+        let mut req = UpgradeRequest {
+            version: "v2".into(),
+            check_sum: "98Ea7aff44631D183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(),
+            image_type: "disk".into(),
+            container_image: "".into(),
+            image_url: "http://localhost:8080/aaa.txt".to_string(),
+            flag_safe: true,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        assert_eq!(handler.paths.image_path.exists(), true);
+        handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum).unwrap();
+
+        req.check_sum = "1234567Abc".into();
+        let res = handler.checksum_match(handler.paths.image_path.to_str().unwrap(), &req.check_sum);
+        assert!(res.is_err());
+    }
+
+    #[test]
+    fn test_load_certs() {
+        init();
+        // This is a tmp cert only for KubeOS unit testing.
+        let tmp_cert = "-----BEGIN CERTIFICATE-----\n\
+        MIIBdDCCARqgAwIBAgIVALnQ5XwM2En1P+xCpkXsO44f8SAUMAoGCCqGSM49BAMC\n\
+        MCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2lnbmVkIGNlcnQwIBcNNzUwMTAxMDAw\n\
+        MDAwWhgPNDA5NjAxMDEwMDAwMDBaMCExHzAdBgNVBAMMFnJjZ2VuIHNlbGYgc2ln\n\
+        bmVkIGNlcnQwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQAi4bkPp5iI9F36HH2\n\
+        Gn+/sC0Ss+DanYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vp\n\
+        HpJNoy0wKzApBgNVHREEIjAgghNoZWxsby53b3JsZC5leGFtcGxlgglsb2NhbGhv\n\
+        c3QwCgYIKoZIzj0EAwIDSAAwRQIhALuS4MU94wJmOZLN+nO7UaTspMN9zbTTkDkG\n\
+        vG+oLD1sAiBg9wpCw+MWJHWvU+H/72mIac9YsC48BYwA7E/LQUOrkw==\n\
+        -----END CERTIFICATE-----\n";
+
+        // This is a tmp private key only for KubeOS unit testing.
+        let tmp_key = "-----BEGIN PRIVATE KEY-----\n\
+        MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQg9Puh/0yMP7S6jXvX\n\
+        Q8K3/COzzyJj84bT8/MJaJ0qp7ihRANCAAQAi4bkPp5iI9F36HH2Gn+/sC0Ss+Da\n\
+        nYY/wEwCrTXDXzAsA0Fuwg0kX75y8qF5JOfWW4tvZwKbeRa5s8vpHpJN\n\
+        -----END PRIVATE KEY-----\n";
+
+        // Create a temporary file to hold the certificate
+        let mut cert_file = NamedTempFile::new().unwrap();
+        cert_file.write_all(tmp_cert.as_bytes()).unwrap();
+
+        // Create a temporary file to hold the private key
+        let mut key_file = NamedTempFile::new().unwrap();
+        key_file.write_all(tmp_key.as_bytes()).unwrap();
+
+        let mut handler = DiskImageHandler::<RealCommandExecutor>::default();
+        handler.certs_path = "".to_string();
+        let certs = CertsInfo {
+            ca_cert: cert_file.path().to_str().unwrap().to_string(),
+            client_cert: cert_file.path().to_str().unwrap().to_string(),
+            client_key: key_file.path().to_str().unwrap().to_string(),
+        };
+
+        let res = handler.load_ca_client_certs(&certs);
+        assert!(res.is_ok());
+
+        let res = handler.load_ca_certs(&certs.ca_cert);
+        assert!(res.is_ok());
+    }
+
+    #[test]
+    fn test_download_image() {
+        init();
+        let tmp_file = NamedTempFile::new().unwrap();
+
+        let mock_executor = MockCommandExec::new();
+        let mut handler = DiskImageHandler::new(PreparePath::default(), mock_executor, String::new());
+        handler.executor.expect_clone().times(1).returning(|| MockCommandExec::new());
+        let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3  ext4\nsda4 /persist ext4\nsr0  iso9660\n";
+        handler.executor.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string()));
+        handler.paths.image_path = tmp_file.path().to_path_buf();
+        assert_eq!(true, handler.paths.image_path.exists());
+
+        let url = mockito::server_url();
+        let upgrade_request = UpgradeRequest {
+            version: "v2".into(),
+            check_sum: "98ea7aff44631d183e6df3488f1107357d7503e11e5f146effdbfd11810cd4a2".into(),
+            image_type: "disk".into(),
+            container_image: "".into(),
+            image_url: format!("{}/test.txt", url),
+            flag_safe: true,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        let _m = mockito::mock("GET", "/test.txt")
+            .with_status(200)
+            .with_body("This is a test txt file for KubeOS test.\n")
+            .create();
+        handler.download_image(&upgrade_request).unwrap();
+        assert_eq!(true, handler.paths.image_path.exists());
+        assert_eq!(
+            fs::read(handler.paths.image_path.to_str().unwrap()).unwrap(),
+            "This is a test txt file for KubeOS test.\n".as_bytes()
+        );
+
+        let _m = mockito::mock("GET", "/test.txt").with_status(404).with_body("Not found").create();
+        let res = handler.download_image(&upgrade_request);
+        assert!(res.is_err())
+    }
+}
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
new file mode 100644
index 0000000000000000000000000000000000000000..4d97552c1bf1ef1acaaf401fc5a2c408ae947c98
--- /dev/null
+++ b/KubeOS-Rust/manager/src/sys_mgmt/docker_image.rs
@@ -0,0 +1,236 @@
+use anyhow::{Context, Result};
+use log::{debug, info, trace};
+
+use crate::{
+    api::{ImageHandler, UpgradeRequest},
+    sys_mgmt::{IMAGE_PERMISSION, NEED_BYTES},
+    utils::*,
+};
+
+pub struct DockerImageHandler<T: CommandExecutor> {
+    pub paths: PreparePath,
+    pub container_name: String,
+    pub executor: T,
+}
+
+impl<T: CommandExecutor> ImageHandler<T> for DockerImageHandler<T> {
+    fn download_image(&self, req: &UpgradeRequest) -> Result<UpgradeImageManager<T>> {
+        perpare_env(&self.paths, NEED_BYTES, IMAGE_PERMISSION)?;
+        self.get_image(req)?;
+        self.get_rootfs_archive(req)?;
+
+        let (_, next_partition_info) = get_partition_info(&self.executor)?;
+        let img_manager = UpgradeImageManager::new(self.paths.clone(), next_partition_info, self.executor.clone());
+        img_manager.create_os_image(IMAGE_PERMISSION)
+    }
+}
+
+impl Default for DockerImageHandler<RealCommandExecutor> {
+    fn default() -> Self {
+        Self { paths: PreparePath::default(), container_name: "kubeos-temp".into(), executor: RealCommandExecutor {} }
+    }
+}
+
+impl<T: CommandExecutor> DockerImageHandler<T> {
+    #[cfg(test)]
+    pub fn new(paths: PreparePath, container_name: String, executor: T) -> Self {
+        Self { paths, container_name, executor }
+    }
+
+    fn get_image(&self, req: &UpgradeRequest) -> Result<()> {
+        let image_name = &req.container_image;
+        is_valid_image_name(image_name)?;
+        let cli = "docker";
+        remove_image_if_exist(cli, image_name, &self.executor)?;
+        info!("Start pulling image {}", image_name);
+        pull_image(cli, image_name, &self.executor)?;
+        info!("Start checking image digest");
+        check_oci_image_digest(cli, image_name, &req.check_sum, &self.executor)?;
+        Ok(())
+    }
+
+    fn get_rootfs_archive(&self, req: &UpgradeRequest) -> Result<()> {
+        let image_name = &req.container_image;
+        info!("Start getting rootfs {}", image_name);
+        self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?;
+        debug!("Create container {}", self.container_name);
+        let container_id =
+            self.executor.run_command_with_output("docker", &["create", "--name", &self.container_name, image_name])?;
+        debug!("Copy rootfs from container {} to {}", container_id, self.paths.update_path.display());
+        self.executor.run_command(
+            "docker",
+            &[
+                "cp",
+                format!("{}:/{}", container_id, self.paths.rootfs_file).as_str(),
+                self.paths.update_path.to_str().unwrap(),
+            ],
+        )?;
+        self.check_and_rm_container().with_context(|| "Failed to remove kubeos-temp container".to_string())?;
+        Ok(())
+    }
+
+    fn check_and_rm_container(&self) -> Result<()> {
+        trace!("Check and remove container {}", self.container_name);
+        let docker_ps_cmd = format!("docker ps -a -f=name={} | awk 'NR==2' | awk '{{print $1}}'", self.container_name);
+        let exist_id = self.executor.run_command_with_output("bash", &["-c", &docker_ps_cmd])?;
+        if !exist_id.is_empty() {
+            info!("Remove container {} {} for cleaning environment", self.container_name, exist_id);
+            self.executor.run_command("docker", &["rm", exist_id.as_str()])?;
+        }
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use mockall::mock;
+
+    use super::*;
+    use crate::api::CertsInfo;
+
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+
+    #[test]
+    fn test_check_and_rm_container() {
+        init();
+        let mut mock_executor = MockCommandExec::new();
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| {
+                cmd == "bash"
+                    && args.len() == 2
+                    && args.contains(&"docker ps -a -f=name=test | awk 'NR==2' | awk '{print $1}'")
+            })
+            .times(1)
+            .returning(|_, _| Ok(String::from("1111")));
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111"))
+            .times(1)
+            .returning(|_, _| Ok(()));
+
+        let result =
+            DockerImageHandler::new(PreparePath::default(), "test".into(), mock_executor).check_and_rm_container();
+        assert!(result.is_ok());
+
+        assert_eq!(DockerImageHandler::default().container_name, "kubeos-temp");
+    }
+
+    #[test]
+    fn test_get_image() {
+        init();
+        let mut mock_executor = MockCommandExec::new();
+        let image_name = "docker.io/library/busybox:latest";
+        let req = UpgradeRequest {
+            version: "KubeOS v2".to_string(),
+            image_type: "docker".to_string(),
+            container_image: image_name.to_string(),
+            check_sum: "22222".to_string(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+
+        // mock remove_image_if_exist
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "docker" && args.contains(&"rmi")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+        // mock pull_image
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| {
+                cmd == "docker" && args.contains(&"pull") && args.contains(&"docker.io/library/busybox:latest")
+            })
+            .times(1)
+            .returning(|_, _| Ok(()));
+        // mock get_oci_image_digest
+        let command_output2 = "[docker.io/library/busybox:latest@sha256:22222]";
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| cmd == "docker" && args.contains(&"inspect") && args.contains(&"{{.RepoDigests}}"))
+            .times(1)
+            .returning(|_, _| Ok(command_output2.to_string()));
+
+        let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor);
+        let result = docker.get_image(&req);
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_get_rootfs_archive() {
+        init();
+        let mut mock_executor = MockCommandExec::new();
+        let image_name = "docker.io/library/busybox:latest";
+        let req = UpgradeRequest {
+            version: "KubeOS v2".to_string(),
+            image_type: "docker".to_string(),
+            container_image: image_name.to_string(),
+            check_sum: "22222".to_string(),
+            image_url: "".to_string(),
+            flag_safe: false,
+            mtls: false,
+            certs: CertsInfo { ca_cert: "".to_string(), client_cert: "".to_string(), client_key: "".to_string() },
+        };
+        // mock check_and_rm_container
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| {
+                cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'")
+            }) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(String::new()));
+        // mock get_rootfs_archive
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| cmd == "docker" && args.contains(&"create")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(String::from("1111")));
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "docker" && args.contains(&"cp") && args.contains(&"1111:/os.tar"))
+            .times(1)
+            .returning(|_, _| Ok(()));
+        // mock check_and_rm_container
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| {
+                cmd == "bash" && args.contains(&"docker ps -a -f=name=kubeos-temp | awk 'NR==2' | awk '{print $1}'")
+            }) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(String::from("1111")));
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "docker" && args.contains(&"rm") && args.contains(&"1111"))
+            .times(1)
+            .returning(|_, _| Ok(()));
+
+        let docker = DockerImageHandler::new(PreparePath::default(), "kubeos-temp".into(), mock_executor);
+        let result = docker.get_rootfs_archive(&req);
+        assert!(result.is_ok());
+    }
+}
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/mod.rs b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..0e06f297b4233d6d7be82b7e68a21e1f26375970
--- /dev/null
+++ b/KubeOS-Rust/manager/src/sys_mgmt/mod.rs
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+mod config;
+mod containerd_image;
+mod disk_image;
+mod docker_image;
+mod values;
+
+pub use config::*;
+pub use containerd_image::*;
+pub use disk_image::*;
+pub use docker_image::*;
+pub use values::*;
diff --git a/KubeOS-Rust/manager/src/sys_mgmt/values.rs b/KubeOS-Rust/manager/src/sys_mgmt/values.rs
new file mode 100644
index 0000000000000000000000000000000000000000..b107efc3b65607620819147145a607d0d5d0addd
--- /dev/null
+++ b/KubeOS-Rust/manager/src/sys_mgmt/values.rs
@@ -0,0 +1,36 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+pub const KERNEL_SYSCTL: &str = "kernel.sysctl";
+pub const KERNEL_SYSCTL_PERSIST: &str = "kernel.sysctl.persist";
+pub const GRUB_CMDLINE_CURRENT: &str = "grub.cmdline.current";
+pub const GRUB_CMDLINE_NEXT: &str = "grub.cmdline.next";
+
+pub const DEFAULT_PROC_PATH: &str = "/proc/sys/";
+pub const DEFAULT_KERNEL_CONFIG_PATH: &str = "/etc/sysctl.conf";
+pub const DEFAULT_GRUB_CFG_PATH: &str = "/boot/efi/EFI/openEuler/grub.cfg";
+pub const DEFAULT_GRUBENV_PATH: &str = "/boot/efi/EFI/openEuler/grubenv";
+
+pub const PERSIST_DIR: &str = "/persist";
+pub const ROOTFS_ARCHIVE: &str = "os.tar";
+pub const UPDATE_DIR: &str = "KubeOS-Update";
+pub const MOUNT_DIR: &str = "kubeos-update";
+pub const OS_IMAGE_NAME: &str = "update.img";
+pub const CERTS_PATH: &str = "/etc/KubeOS/certs/";
+
+pub const DEFAULT_KERNEL_CONFIG_PERM: u32 = 0o644;
+pub const DEFAULT_GRUB_CFG_PERM: u32 = 0o751;
+pub const IMAGE_PERMISSION: u32 = 0o600;
+
+pub const ONLY_KEY: usize = 1;
+pub const KV_PAIR: usize = 2;
+pub const NEED_BYTES: i64 = 3 * 1024 * 1024 * 1024;
diff --git a/KubeOS-Rust/manager/src/utils/common.rs b/KubeOS-Rust/manager/src/utils/common.rs
new file mode 100644
index 0000000000000000000000000000000000000000..9baf99e350f19a912479e08388a31f5b1c4ac7fe
--- /dev/null
+++ b/KubeOS-Rust/manager/src/utils/common.rs
@@ -0,0 +1,310 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{
+    fs,
+    os::{linux::fs::MetadataExt, unix::fs::DirBuilderExt},
+    path::{Path, PathBuf},
+};
+
+use anyhow::{anyhow, bail, Context, Result};
+use log::{debug, info, trace};
+use nix::{mount, mount::MntFlags};
+
+use super::executor::CommandExecutor;
+use crate::sys_mgmt::{MOUNT_DIR, OS_IMAGE_NAME, PERSIST_DIR, ROOTFS_ARCHIVE, UPDATE_DIR};
+
+/// * persist_path: /persist
+///
+/// * update_path: /persist/KubeOS-Update
+///
+/// * mount_path: /persist/KubeOS-Update/kubeos-update
+///
+/// * tar_path: /persist/KubeOS-Update/os.tar
+///
+/// * image_path: /persist/update.img
+///
+/// * rootfs_file: os.tar
+#[derive(Clone)]
+pub struct PreparePath {
+    pub persist_path: PathBuf,
+    pub update_path: PathBuf,
+    pub mount_path: PathBuf,
+    pub tar_path: PathBuf,
+    pub image_path: PathBuf,
+    pub rootfs_file: String,
+}
+
+impl Default for PreparePath {
+    fn default() -> Self {
+        let persist_dir = Path::new(PERSIST_DIR);
+        let update_pathbuf = persist_dir.join(UPDATE_DIR);
+        Self {
+            persist_path: persist_dir.to_path_buf(),
+            update_path: update_pathbuf.clone(),
+            mount_path: update_pathbuf.join(MOUNT_DIR),
+            tar_path: update_pathbuf.join(ROOTFS_ARCHIVE),
+            image_path: persist_dir.join(OS_IMAGE_NAME),
+            rootfs_file: ROOTFS_ARCHIVE.to_string(),
+        }
+    }
+}
+
+pub fn is_file_exist<P: AsRef<Path>>(path: P) -> bool {
+    path.as_ref().exists()
+}
+
+pub fn perpare_env(prepare_path: &PreparePath, need_bytes: i64, permission: u32) -> Result<()> {
+    info!("Prepare environment to upgrade");
+    check_disk_size(need_bytes, &prepare_path.persist_path)?;
+    clean_env(&prepare_path.update_path, &prepare_path.mount_path, &prepare_path.image_path)?;
+    fs::DirBuilder::new().recursive(true).mode(permission).create(&prepare_path.mount_path)?;
+    Ok(())
+}
+
+pub fn check_disk_size<P: AsRef<Path>>(need_bytes: i64, path: P) -> Result<()> {
+    trace!("Check if there is enough disk space to upgrade");
+    let fs_stat = nix::sys::statfs::statfs(path.as_ref())?;
+    let available_blocks = i64::try_from(fs_stat.blocks_available())?;
+    let available_space = available_blocks * fs_stat.block_size();
+    if available_space < need_bytes {
+        bail!("Space is not enough for downloading");
+    }
+    Ok(())
+}
+
+/// clean_env will umount the mount path and delete directory /persist/KubeOS-Update and /persist/update.img
+pub fn clean_env<P>(update_path: P, mount_path: P, image_path: P) -> Result<()>
+where
+    P: AsRef<Path> + std::fmt::Debug,
+{
+    if is_mounted(&mount_path)? {
+        debug!("Umount \"{}\"", mount_path.as_ref().display());
+        if let Err(errno) = mount::umount2(mount_path.as_ref(), MntFlags::MNT_FORCE) {
+            bail!("Failed to umount {} in clean_env: {}", mount_path.as_ref().display(), errno);
+        }
+    }
+    // losetup -D?
+    delete_file_or_dir(&update_path).with_context(|| format!("Failed to delete {:?}", update_path))?;
+    delete_file_or_dir(&image_path).with_context(|| format!("Failed to delete {:?}", image_path))?;
+    Ok(())
+}
+
+pub fn delete_file_or_dir<P: AsRef<Path>>(path: P) -> Result<()> {
+    if is_file_exist(&path) {
+        if fs::metadata(&path)?.is_file() {
+            info!("Delete file \"{}\"", path.as_ref().display());
+            fs::remove_file(&path)?;
+        } else {
+            info!("Delete directory \"{}\"", path.as_ref().display());
+            fs::remove_dir_all(&path)?;
+        }
+    }
+    Ok(())
+}
+
+pub fn is_command_available<T: CommandExecutor>(command: &str, command_executor: &T) -> bool {
+    match command_executor.run_command("/bin/sh", &["-c", format!("command -v {}", command).as_str()]) {
+        Ok(_) => {
+            debug!("command {} is available", command);
+            true
+        },
+        Err(_) => {
+            debug!("command {} is not available", command);
+            false
+        },
+    }
+}
+
+pub fn is_mounted<P: AsRef<Path>>(mount_path: P) -> Result<bool> {
+    if !is_file_exist(&mount_path) {
+        return Ok(false);
+    }
+    // Get device ID of mountPath
+    let mount_meta = fs::symlink_metadata(&mount_path)?;
+    let dev = mount_meta.st_dev();
+
+    // Get device ID of mountPath's parent directory
+    let parent = mount_path
+        .as_ref()
+        .parent()
+        .ok_or_else(|| anyhow!("Failed to get parent directory of {}", mount_path.as_ref().display()))?;
+    let parent_meta = fs::symlink_metadata(parent)?;
+    let dev_parent = parent_meta.st_dev();
+    Ok(dev != dev_parent)
+}
+
+pub fn switch_boot_menuentry<T: CommandExecutor>(
+    command_executor: &T,
+    grub_env_path: &str,
+    next_menuentry: &str,
+) -> Result<()> {
+    if get_boot_mode() == "uefi" {
+        command_executor.run_command(
+            "grub2-editenv",
+            &[grub_env_path, "set", format!("saved_entry={}", next_menuentry).as_str()],
+        )?;
+    } else {
+        command_executor.run_command("grub2-set-default", &[next_menuentry])?;
+    }
+    Ok(())
+}
+
+pub fn get_boot_mode() -> String {
+    if is_file_exist("/sys/firmware/efi") { "uefi".into() } else { "bios".into() }
+}
+
+#[cfg(test)]
+mod tests {
+    use mockall::{mock, predicate::*};
+    use tempfile::{NamedTempFile, TempDir};
+
+    use super::*;
+    use crate::utils::RealCommandExecutor;
+
+    // Mock the CommandExecutor trait
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+
+    #[test]
+    fn test_is_file_exist() {
+        init();
+        let path = "/tmp/test_is_file_exist";
+        assert_eq!(is_file_exist(path), false);
+
+        let file = NamedTempFile::new().unwrap();
+        assert_eq!(is_file_exist(file.path().to_str().unwrap()), true);
+
+        let tmp_dir = TempDir::new().unwrap();
+        assert_eq!(is_file_exist(tmp_dir.path().to_str().unwrap()), true);
+    }
+
+    #[test]
+    fn test_prepare_env() {
+        init();
+        let paths = PreparePath {
+            persist_path: PathBuf::from("/tmp"),
+            update_path: PathBuf::from("/tmp/test_prepare_env"),
+            mount_path: PathBuf::from("/tmp/test_prepare_env/kubeos-update"),
+            tar_path: PathBuf::from("/tmp/test_prepare_env/os.tar"),
+            image_path: PathBuf::from("/tmp/test_prepare_env/update.img"),
+            rootfs_file: "os.tar".to_string(),
+        };
+        perpare_env(&paths, 1 * 1024 * 1024 * 1024, 0o700).unwrap();
+    }
+
+    #[test]
+    fn test_check_disk_size() {
+        init();
+        let path = "/home";
+        let gb: i64 = 1 * 1024 * 1024 * 1024;
+        let need_gb = 1 * gb;
+        let result = check_disk_size(need_gb, path);
+        assert!(result.is_ok());
+        let need_gb = 10000 * gb;
+        let result = check_disk_size(need_gb, path);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_clean_env() {
+        init();
+        let update_path = "/tmp/test_clean_env";
+        let mount_path = "/tmp/test_clean_env/kubeos-update";
+        let image_path = "/tmp/test_clean_env/update.img";
+        clean_env(&update_path.to_string(), &mount_path.to_string(), &image_path.to_string()).unwrap();
+    }
+
+    #[test]
+    fn test_delete_file_or_dir() {
+        init();
+        let path = "/tmp/test_delete_file";
+        fs::File::create(path).unwrap();
+        assert_eq!(Path::new(path).exists(), true);
+        delete_file_or_dir(&path.to_string()).unwrap();
+        assert_eq!(Path::new(path).exists(), false);
+
+        let path = "/tmp/test_dir";
+        fs::create_dir(path).unwrap();
+        assert_eq!(Path::new(path).exists(), true);
+        delete_file_or_dir(&path.to_string()).unwrap();
+        assert_eq!(Path::new(path).exists(), false);
+
+        let path = "/tmp/nonexist";
+        delete_file_or_dir(path).unwrap();
+
+        let path = PathBuf::new();
+        delete_file_or_dir(path).unwrap();
+    }
+
+    #[test]
+    fn test_switch_boot_menuentry() {
+        init();
+        let grubenv_path = "/boot/efi/EFI/openEuler/grubenv";
+        let next_menuentry = "B";
+        let mut mock = MockCommandExec::new();
+        if get_boot_mode() == "uefi" {
+            mock.expect_run_command()
+                .withf(move |name, args| {
+                    name == "grub2-editenv"
+                        && args[0] == grubenv_path
+                        && args[2] == format!("saved_entry={}", next_menuentry).as_str()
+                })
+                .times(1) // Expect it to be called once
+                .returning(move |_, _| Ok(()));
+        } else {
+            mock.expect_run_command()
+                .withf(move |name, args| name == "grub2-set-default" && args[0] == next_menuentry)
+                .times(1) // Expect it to be called once
+                .returning(move |_, _| Ok(()));
+        }
+
+        switch_boot_menuentry(&mock, grubenv_path, next_menuentry).unwrap()
+    }
+
+    #[test]
+    fn test_get_boot_mode() {
+        init();
+        let boot_mode = get_boot_mode();
+        let executor = RealCommandExecutor {};
+        let res = executor.run_command("ls", &["/sys/firmware/efi"]);
+        if res.is_ok() {
+            assert!(boot_mode == "uefi");
+        } else {
+            assert!(boot_mode == "bios");
+        }
+    }
+
+    #[test]
+    fn test_is_command_available() {
+        init();
+        let executor = RealCommandExecutor {};
+        assert_eq!(is_command_available("ls", &executor), true);
+        assert_eq!(is_command_available("aaaabb", &executor), false);
+    }
+}
diff --git a/KubeOS-Rust/manager/src/utils/container_image.rs b/KubeOS-Rust/manager/src/utils/container_image.rs
new file mode 100644
index 0000000000000000000000000000000000000000..d84c6853a3431ca85c7ba1872e7ee94b78284a2b
--- /dev/null
+++ b/KubeOS-Rust/manager/src/utils/container_image.rs
@@ -0,0 +1,341 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use anyhow::{bail, Result};
+use log::{debug, info, trace};
+use regex::Regex;
+
+use super::executor::CommandExecutor;
+
+pub fn is_valid_image_name(image: &str) -> Result<()> {
+    let pattern = r"^((?:[\w.-]+)(?::\d+)?/)*(?:[\w.-]+)((?::[\w_.-]+)?|(?:@sha256:[a-fA-F0-9]+)?)$";
+    let reg_ex = Regex::new(pattern)?;
+    if !reg_ex.is_match(image) {
+        bail!("Invalid image name: {}", image);
+    }
+    debug!("Image name {} is valid", image);
+    Ok(())
+}
+
+pub fn check_oci_image_digest<T: CommandExecutor>(
+    container_runtime: &str,
+    image_name: &str,
+    check_sum: &str,
+    command_executor: &T,
+) -> Result<()> {
+    let image_digests = get_oci_image_digest(container_runtime, image_name, command_executor)?;
+    if image_digests.to_lowercase() != check_sum.to_lowercase() {
+        bail!("Image digest mismatch, expect {}, got {}", check_sum, image_digests);
+    }
+    Ok(())
+}
+
+pub fn get_oci_image_digest<T: CommandExecutor>(
+    container_runtime: &str,
+    image_name: &str,
+    executor: &T,
+) -> Result<String> {
+    let cmd_output: String;
+    match container_runtime {
+        "crictl" => {
+            cmd_output = executor.run_command_with_output(
+                "crictl",
+                &["inspecti", "--output", "go-template", "--template", "{{.status.repoDigests}}", image_name],
+            )?;
+        },
+        "docker" => {
+            cmd_output =
+                executor.run_command_with_output("docker", &["inspect", "--format", "{{.RepoDigests}}", image_name])?;
+        },
+        "ctr" => {
+            cmd_output = executor
+                .run_command_with_output("ctr", &["-n", "k8s.io", "images", "ls", &format!("name=={}", image_name)])?;
+            // Split by whitespaces, we get vec like [REF TYPE DIGEST SIZE PLATFORMS LABELS x x x x x x]
+            // get the 8th element, and split by ':' to get the digest
+            let fields: Vec<&str> = cmd_output.split_whitespace().collect();
+            if let Some(digest) = fields.get(8).and_then(|field| field.split(':').nth(1)) {
+                trace!("get_oci_image_digest: {}", digest);
+                return Ok(digest.to_string());
+            } else {
+                bail!("Failed to get digest from ctr command output: {}", cmd_output);
+            }
+        },
+        _ => {
+            bail!("Container runtime {} cannot be recognized", container_runtime);
+        },
+    }
+
+    // Parse the cmd_output to extract the digest
+    let parts: Vec<&str> = cmd_output.split('@').collect();
+    if let Some(last_part) = parts.last() {
+        if last_part.starts_with("sha256") {
+            let parsed_parts: Vec<&str> = last_part.trim_matches(|c| c == ']').split(':').collect();
+            // After spliiing by ':', we should get vec like [sha256, digests]
+            if parsed_parts.len() == 2 {
+                debug!("get_oci_image_digest: {}", parsed_parts[1]);
+                return Ok(parsed_parts[1].to_string()); // 1 is the index of digests
+            }
+        }
+    }
+
+    bail!("Failed to get digest from command output: {}", cmd_output)
+}
+
+pub fn pull_image<T: CommandExecutor>(runtime: &str, image_name: &str, executor: &T) -> Result<()> {
+    debug!("Pull image {}", image_name);
+    match runtime {
+        "crictl" => {
+            executor.run_command("crictl", &["pull", image_name])?;
+        },
+        "ctr" => {
+            executor.run_command(
+                "ctr",
+                &[&"-n", "k8s.io", "images", "pull", "--hosts-dir", "/etc/containerd/certs.d", image_name],
+            )?;
+        },
+        "docker" => {
+            executor.run_command("docker", &["pull", image_name])?;
+        },
+        _ => {
+            bail!("Container runtime {} cannot be recognized", runtime);
+        },
+    }
+    Ok(())
+}
+
+pub fn remove_image_if_exist<T: CommandExecutor>(runtime: &str, image_name: &str, executor: &T) -> Result<()> {
+    match runtime {
+        "crictl" => {
+            if executor.run_command("crictl", &["inspecti", image_name]).is_ok() {
+                executor.run_command("crictl", &["rmi", image_name])?;
+                info!("Remove existing upgrade image: {}", image_name);
+            }
+        },
+        "ctr" => {
+            let output = executor.run_command_with_output(
+                "ctr",
+                &[&"-n", "k8s.io", "images", "check", &format!("name=={}", image_name)],
+            )?;
+            if !output.is_empty() {
+                executor.run_command("ctr", &[&"-n", "k8s.io", "images", "rm", image_name, "--sync"])?;
+                info!("Remove existing upgrade image: {}", image_name);
+            }
+        },
+        "docker" => {
+            if executor.run_command("docker", &["inspect", image_name]).is_ok() {
+                executor.run_command("docker", &["rmi", image_name])?;
+                info!("Remove existing upgrade image: {}", image_name);
+            }
+        },
+        _ => {
+            bail!("Container runtime {} cannot be recognized", runtime);
+        },
+    }
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use mockall::{mock, predicate::*};
+
+    use super::*;
+
+    // Mock the CommandExecutor trait
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+
+    #[test]
+    fn test_is_valid_image_name() {
+        init();
+        let correct_images = vec![
+            "alpine",
+            "alpine:latest",
+            "localhost/latest",
+            "library/alpine",
+            "localhost:1234/test",
+            "test:1234/blaboon",
+            "alpine:3.7",
+            "docker.example.edu/gmr/alpine:3.7",
+            "docker.example.com:5000/gmr/alpine@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8",
+            "docker.example.co.uk/gmr/alpine/test2:latest",
+            "registry.dobby.org/dobby/dobby-servers/arthound:2019-08-08",
+            "owasp/zap:3.8.0",
+            "registry.dobby.co/dobby/dobby-servers/github-run:2021-10-04",
+            "docker.elastic.co/kibana/kibana:7.6.2",
+            "registry.dobby.org/dobby/dobby-servers/lerphound:latest",
+            "registry.dobby.org/dobby/dobby-servers/marbletown-poc:2021-03-29",
+            "marbles/marbles:v0.38.1",
+            "registry.dobby.org/dobby/dobby-servers/loophole@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8",
+            "sonatype/nexon:3.30.0",
+            "prom/node-exporter:v1.1.1",
+            "sosedoff/pgweb@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04abc574c8",
+            "sosedoff/pgweb:latest",
+            "registry.dobby.org/dobby/dobby-servers/arpeggio:2021-06-01",
+            "registry.dobby.org/dobby/antique-penguin:release-production",
+            "dalprodictus/halcon:6.7.5",
+            "antigua/antigua:v31",
+            "weblate/weblate:4.7.2-1",
+            "redis:4.0.01-alpine",
+            "registry.dobby.com/dobby/dobby-servers/github-run:latest",
+            "192.168.122.123:5000/kubeos-x86_64:2023-01",
+        ];
+        let wrong_images = vec![
+            "alpine;v1.0",
+            "alpine:latest@sha256:11111111111111111111111111111111",
+            "alpine|v1.0",
+            "alpine&v1.0",
+            "sosedoff/pgweb:latest@sha256:5a156ff125e5a12ac7ff43ee5120fa249cf62248337b6d04574c8",
+            "192.168.122.123:5000/kubeos-x86_64:2023-01@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a",
+            "192.168.122.123:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a",
+            "myimage$%^&",
+            ":myimage",
+            "/myimage",
+            "myimage/",
+            "myimage:",
+            "myimage@@latest",
+            "myimage::tag",
+            "registry.com//myimage:tag",
+            " myimage",
+            "myimage ",
+            "registry.com/:tag",
+            "myimage:",
+            "",
+            ":tag",
+            "IP:5000@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a",
+        ];
+        for image in correct_images {
+            assert!(is_valid_image_name(image).is_ok());
+        }
+        for image in wrong_images {
+            assert!(is_valid_image_name(image).is_err());
+        }
+    }
+
+    #[test]
+    fn test_get_oci_image_digest() {
+        init();
+        let mut mock = MockCommandExec::new();
+        let container_runtime = "ctr";
+        let image_name = "docker.io/nginx:latest";
+        let command_output1 =
+            "REF TYPE DIGEST SIZE PLATFORMS LABELS\ndocker.io/nginx:latest text/html sha256:1111 132.5 KIB - -\n";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string()));
+        let out1 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap();
+        let expect_output = "1111";
+        assert_eq!(out1, expect_output);
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok("invalid output".to_string()));
+        let out2 = get_oci_image_digest(container_runtime, image_name, &mock);
+        assert!(out2.is_err());
+
+        let container_runtime = "crictl";
+        let command_output2 = "[docker.io/nginx@sha256:1111]";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string()));
+        let out3 = get_oci_image_digest(container_runtime, image_name, &mock).unwrap();
+        assert_eq!(out3, expect_output);
+
+        let out4 = get_oci_image_digest("invalid", image_name, &mock);
+        assert!(out4.is_err());
+
+        let container_runtime = "crictl";
+        let command_output3 = "[docker.io/nginx:sha256:1111]";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string()));
+        let out5 = get_oci_image_digest(container_runtime, image_name, &mock);
+        assert!(out5.is_err());
+    }
+
+    #[test]
+    fn test_check_oci_image_digest_match() {
+        init();
+        let mut mock = MockCommandExec::new();
+        let image_name = "docker.io/nginx:latest";
+        let container_runtime = "crictl";
+        let command_output = "[docker.io/nginx@sha256:1a2b]";
+        let check_sum = "1A2B";
+        mock.expect_run_command_with_output().times(2).returning(|_, _| Ok(command_output.to_string()));
+        let result = check_oci_image_digest(container_runtime, image_name, check_sum, &mock);
+        assert!(result.is_ok());
+        let result = check_oci_image_digest(container_runtime, image_name, "1111", &mock);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_pull_image() {
+        init();
+        let mut mock_executor = MockCommandExec::new();
+
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "crictl" && args.len() == 2 && args[0] == "pull") // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "ctr" && args.len() == 7 && args[3] == "pull") // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "docker" && args.len() == 2 && args[0] == "pull") // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+
+        let image_name = "docker.io/nginx:latest";
+        let result = pull_image("crictl", image_name, &mock_executor);
+        assert!(result.is_ok());
+        let result = pull_image("ctr", image_name, &mock_executor);
+        assert!(result.is_ok());
+        let result = pull_image("docker", image_name, &mock_executor);
+        assert!(result.is_ok());
+        let result = pull_image("aaa", image_name, &mock_executor);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_remove_image_if_exist() {
+        init();
+        let mut mock_executor = MockCommandExec::new();
+        mock_executor
+            .expect_run_command_with_output()
+            .withf(|cmd, args| cmd == "ctr" && args.contains(&"check")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(String::from("something")));
+        mock_executor
+            .expect_run_command()
+            .withf(|cmd, args| cmd == "ctr" && args.contains(&"rm")) // simplified with a closure
+            .times(1)
+            .returning(|_, _| Ok(()));
+        let image_name = "docker.io/nginx:latest";
+        let res = remove_image_if_exist("ctr", image_name, &mock_executor);
+        assert!(res.is_ok());
+
+        let res = remove_image_if_exist("invalid", image_name, &mock_executor);
+        assert!(res.is_err());
+    }
+}
diff --git a/KubeOS-Rust/manager/src/utils/executor.rs b/KubeOS-Rust/manager/src/utils/executor.rs
new file mode 100644
index 0000000000000000000000000000000000000000..c87bf2ad42316a63d42f1104f56efe24af369e72
--- /dev/null
+++ b/KubeOS-Rust/manager/src/utils/executor.rs
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::process::Command;
+
+use anyhow::{bail, Result};
+use log::{debug, trace};
+
+pub trait CommandExecutor: Clone {
+    fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+    fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+}
+
+#[derive(Clone)]
+pub struct RealCommandExecutor {}
+
+impl CommandExecutor for RealCommandExecutor {
+    fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()> {
+        trace!("run_command: {} {:?}", name, args);
+        let output = Command::new(name).args(args).output()?;
+        if !output.status.success() {
+            let stdout = String::from_utf8_lossy(&output.stdout);
+            let error_message = String::from_utf8_lossy(&output.stderr);
+            bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message);
+        }
+        debug!("run_command: {} {:?} done", name, args);
+        Ok(())
+    }
+
+    fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String> {
+        trace!("run_command_with_output: {} {:?}", name, args);
+        let output = Command::new(name).args(args).output()?;
+        let stdout = String::from_utf8_lossy(&output.stdout).to_string();
+        if !output.status.success() {
+            let error_message = String::from_utf8_lossy(&output.stderr);
+            bail!("Failed to run command: {} {:?}, stdout: \"{}\", stderr: \"{}\"", name, args, stdout, error_message);
+        }
+        debug!("run_command_with_output: {} {:?} done", name, args);
+        Ok(stdout.trim_end_matches('\n').to_string())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+
+    #[test]
+    fn test_run_command_with_output() {
+        init();
+        let executor: RealCommandExecutor = RealCommandExecutor {};
+
+        // test run_command_with_output
+        let output = executor.run_command_with_output("echo", &["hello", "world"]).unwrap();
+        assert_eq!(output, "hello world");
+        let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "cat").as_str()]).unwrap();
+        assert_eq!(out, "/usr/bin/cat");
+        let out = executor.run_command_with_output("sh", &["-c", format!("command -v {}", "apple").as_str()]);
+        assert!(out.is_err());
+    }
+
+    #[test]
+    fn test_run_command() {
+        init();
+        let executor: RealCommandExecutor = RealCommandExecutor {};
+        // test run_command
+        let out = executor.run_command("sh", &["-c", format!("command -v {}", "apple").as_str()]);
+        assert!(out.is_err());
+
+        let out = executor.run_command("sh", &["-c", format!("command -v {}", "cat").as_str()]);
+        assert!(out.is_ok());
+    }
+}
diff --git a/KubeOS-Rust/manager/src/utils/image_manager.rs b/KubeOS-Rust/manager/src/utils/image_manager.rs
new file mode 100644
index 0000000000000000000000000000000000000000..90806cf89bc2f21580861fa5e8aedd3e95b31d1a
--- /dev/null
+++ b/KubeOS-Rust/manager/src/utils/image_manager.rs
@@ -0,0 +1,206 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{
+    fs::{self, Permissions},
+    os::unix::fs::PermissionsExt,
+    path::PathBuf,
+};
+
+use anyhow::{Context, Result};
+use log::{debug, info};
+
+use super::{
+    clean_env,
+    common::{delete_file_or_dir, PreparePath},
+    executor::CommandExecutor,
+    partition::PartitionInfo,
+};
+
+pub struct UpgradeImageManager<T: CommandExecutor> {
+    pub paths: PreparePath,
+    pub next_partition: PartitionInfo,
+    pub executor: T,
+}
+
+impl<T: CommandExecutor> UpgradeImageManager<T> {
+    pub fn new(paths: PreparePath, next_partition: PartitionInfo, executor: T) -> Self {
+        Self { paths, next_partition, executor }
+    }
+
+    fn image_path_str(&self) -> Result<&str> {
+        self.paths.image_path.to_str().context("Failed to convert image path to string")
+    }
+
+    fn mount_path_str(&self) -> Result<&str> {
+        self.paths.mount_path.to_str().context("Failed to convert mount path to string")
+    }
+
+    fn tar_path_str(&self) -> Result<&str> {
+        self.paths.tar_path.to_str().context("Failed to convert tar path to string")
+    }
+
+    pub fn create_image_file(&self, permission: u32) -> Result<()> {
+        let image_str = self.image_path_str()?;
+
+        debug!("Create image {}", image_str);
+        self.executor.run_command("dd", &["if=/dev/zero", &format!("of={}", image_str), "bs=2M", "count=1024"])?;
+        fs::set_permissions(&self.paths.image_path, Permissions::from_mode(permission))?;
+        Ok(())
+    }
+
+    pub fn format_image(&self) -> Result<()> {
+        let image_str = self.image_path_str()?;
+        debug!("Format image {}", image_str);
+        self.executor.run_command(
+            format!("mkfs.{}", self.next_partition.fs_type).as_str(),
+            &["-L", format!("ROOT-{}", self.next_partition.menuentry).as_str(), image_str],
+        )?;
+        Ok(())
+    }
+
+    pub fn mount_image(&self) -> Result<()> {
+        let image_str = self.image_path_str()?;
+        let mount_str = self.mount_path_str()?;
+        debug!("Mount {} to {}", image_str, mount_str);
+        self.executor.run_command("mount", &["-o", "loop", image_str, mount_str])?;
+        Ok(())
+    }
+
+    pub fn extract_tar_to_image(&self) -> Result<()> {
+        let tar_str = self.tar_path_str()?;
+        let mount_str = self.mount_path_str()?;
+        debug!("Extract {} to mounted path {}", tar_str, mount_str);
+        self.executor.run_command("tar", &["-xvf", tar_str, "-C", mount_str])?;
+        Ok(())
+    }
+
+    pub fn create_os_image(self, permission: u32) -> Result<Self> {
+        self.create_image_file(permission)?;
+        self.format_image()?;
+        self.mount_image()?;
+        self.extract_tar_to_image()?;
+        // Pass empty image_path to clean_env but avoid deleting the upgrade image
+        clean_env(&self.paths.update_path, &self.paths.mount_path, &PathBuf::new())?;
+        Ok(self)
+    }
+
+    pub fn install(&self) -> Result<()> {
+        let image_str = self.image_path_str()?;
+        let device = self.next_partition.device.as_str();
+        self.executor
+            .run_command("dd", &[format!("if={}", image_str).as_str(), format!("of={}", device).as_str(), "bs=8M"])?;
+        debug!("Install image {} to {} done", image_str, device);
+        info!(
+            "Device {} is overwritten and unable to rollback to the previous version anymore if the eviction of node fails",
+            device
+        );
+        delete_file_or_dir(image_str)?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::{fs, io::Write, path::Path};
+
+    use mockall::{mock, predicate::*};
+    use tempfile::NamedTempFile;
+
+    use super::*;
+
+    // Mock the CommandExecutor trait
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+
+    #[test]
+    fn test_update_image_manager() {
+        init();
+        // create a dir in tmp dir
+        let tmp_dir = "/tmp/test_update_image_manager";
+        let img_path = format!("{}/test_image", tmp_dir);
+        let mut temp_file = NamedTempFile::new().unwrap();
+        write!(temp_file, "test content").unwrap(); // Writing s
+        fs::create_dir(tmp_dir).unwrap();
+        let clone_img_path = img_path.clone();
+
+        let mut mock = MockCommandExec::new();
+        //mock create_image_file
+        mock.expect_run_command()
+            .withf(|name, args| name == "dd" && args[0] == "if=/dev/zero")
+            .times(1) // Expect it to be called once
+            .returning(move |_, _| {
+                // simulate 'dd' by copying the contents of the temporary file
+                std::fs::copy(temp_file.path(), &clone_img_path).unwrap();
+                Ok(())
+            });
+
+        //mock format_image
+        mock.expect_run_command()
+            .withf(|name, args| name == "mkfs.ext4" && args[1] == "ROOT-B")
+            .times(1) // Expect it to be called once
+            .returning(|_, _| Ok(()));
+
+        //mock mount_image
+        mock.expect_run_command()
+            .withf(|name, _| name == "mount")
+            .times(1) // Expect it to be called once
+            .returning(|_, _| Ok(()));
+
+        //mock extract_tar_to_image
+        mock.expect_run_command()
+            .withf(|name, args| name == "tar" && args[0] == "-xvf")
+            .times(1) // Expect it to be called once
+            .returning(|_, _| Ok(()));
+
+        //mock install->dd
+        mock.expect_run_command()
+            .withf(|name, _| name == "dd")
+            .times(1) // Expect it to be called once
+            .returning(|_, _| Ok(()));
+
+        let img_manager = UpgradeImageManager::new(
+            PreparePath {
+                persist_path: "/tmp".into(),
+                update_path: tmp_dir.into(),
+                image_path: img_path.into(),
+                mount_path: "/tmp/update/mount".into(),
+                tar_path: "/tmp/update/image.tar".into(),
+                rootfs_file: "image.tar".into(),
+            },
+            PartitionInfo { device: "/dev/sda3".into(), fs_type: "ext4".into(), menuentry: "B".into() },
+            mock,
+        );
+
+        let img_manager = img_manager.create_os_image(0o755).unwrap();
+        let result = img_manager.install();
+        assert!(result.is_ok());
+
+        assert_eq!(Path::new(&tmp_dir).exists(), false);
+    }
+}
diff --git a/KubeOS-Rust/manager/src/utils/mod.rs b/KubeOS-Rust/manager/src/utils/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..caf406e3229666662c3026c4786b9ebf85b3f806
--- /dev/null
+++ b/KubeOS-Rust/manager/src/utils/mod.rs
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+mod common;
+mod container_image;
+mod executor;
+mod image_manager;
+mod partition;
+
+pub use common::*;
+pub use container_image::*;
+pub use executor::*;
+pub use image_manager::*;
+pub use partition::*;
diff --git a/KubeOS-Rust/manager/src/utils/partition.rs b/KubeOS-Rust/manager/src/utils/partition.rs
new file mode 100644
index 0000000000000000000000000000000000000000..799b4b35d63bdcfbe06c0c620dca5e68d7f5b464
--- /dev/null
+++ b/KubeOS-Rust/manager/src/utils/partition.rs
@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use anyhow::{bail, Result};
+use log::{debug, trace};
+
+use super::executor::CommandExecutor;
+
+#[derive(PartialEq, Debug, Default)]
+pub struct PartitionInfo {
+    pub device: String,
+    pub menuentry: String,
+    pub fs_type: String,
+}
+
+/// get_partition_info returns the current partition info and the next partition info.
+pub fn get_partition_info<T: CommandExecutor>(executor: &T) -> Result<(PartitionInfo, PartitionInfo), anyhow::Error> {
+    let lsblk = executor.run_command_with_output("lsblk", &["-lno", "NAME,MOUNTPOINTS,FSTYPE"])?;
+    // After split whitespace, the root directory line should have 3 elements, which are "sda2 / ext4".
+    let mut cur_partition = PartitionInfo::default();
+    let mut next_partition = PartitionInfo::default();
+    let splitted_len = 3;
+    trace!("get_partition_info lsblk command output:\n{}", lsblk);
+    for line in lsblk.lines() {
+        let res: Vec<&str> = line.split_whitespace().collect();
+        if res.len() == splitted_len && res[1] == "/" {
+            debug!("root directory line: device={}, fs_type={}", res[0], res[2]);
+            cur_partition.device = format!("/dev/{}", res[0]).to_string();
+            cur_partition.fs_type = res[2].to_string();
+            next_partition.fs_type = res[2].to_string();
+            if res[0].contains('2') {
+                // root directory is mounted on sda2, so sda3 is the next partition
+                cur_partition.menuentry = String::from("A");
+                next_partition.menuentry = String::from("B");
+                next_partition.device = format!("/dev/{}", res[0].replace('2', "3")).to_string();
+            } else if res[0].contains('3') {
+                // root directory is mounted on sda3, so sda2 is the next partition
+                cur_partition.menuentry = String::from("B");
+                next_partition.menuentry = String::from("A");
+                next_partition.device = format!("/dev/{}", res[0].replace('3', "2")).to_string();
+            }
+        }
+    }
+    if cur_partition.menuentry.is_empty() {
+        bail!("Failed to get partition info, lsblk output: {}", lsblk);
+    }
+    Ok((cur_partition, next_partition))
+}
+
+#[cfg(test)]
+mod tests {
+    use mockall::{mock, predicate::*};
+
+    use super::*;
+
+    // Mock the CommandExecutor trait
+    mock! {
+        pub CommandExec{}
+        impl CommandExecutor for CommandExec {
+            fn run_command<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<()>;
+            fn run_command_with_output<'a>(&self, name: &'a str, args: &[&'a str]) -> Result<String>;
+        }
+        impl Clone for CommandExec {
+            fn clone(&self) -> Self;
+        }
+    }
+
+    fn init() {
+        let _ = env_logger::builder()
+            .target(env_logger::Target::Stdout)
+            .filter_level(log::LevelFilter::Trace)
+            .is_test(true)
+            .try_init();
+    }
+
+    #[test]
+    fn test_get_partition_info() {
+        init();
+        let command_output1 = "sda\nsda1 /boot/efi vfat\nsda2 / ext4\nsda3  ext4\nsda4 /persist ext4\nsr0  iso9660\n";
+        let mut mock = MockCommandExec::new();
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output1.to_string()));
+        let res = get_partition_info(&mock).unwrap();
+        let expect_res = (
+            PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() },
+            PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() },
+        );
+        assert_eq!(res, expect_res);
+
+        let command_output2 = "sda\nsda1 /boot/efi vfat\nsda2   ext4\nsda3 / ext4\nsda4 /persist ext4\nsr0  iso9660\n";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output2.to_string()));
+        let res = get_partition_info(&mock).unwrap();
+        let expect_res = (
+            PartitionInfo { device: "/dev/sda3".to_string(), menuentry: "B".to_string(), fs_type: "ext4".to_string() },
+            PartitionInfo { device: "/dev/sda2".to_string(), menuentry: "A".to_string(), fs_type: "ext4".to_string() },
+        );
+        assert_eq!(res, expect_res);
+
+        let command_output3 = "";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output3.to_string()));
+        let res = get_partition_info(&mock);
+        assert!(res.is_err());
+
+        let command_output4 = "sda4 / ext4";
+        mock.expect_run_command_with_output().times(1).returning(|_, _| Ok(command_output4.to_string()));
+        let res = get_partition_info(&mock);
+        assert!(res.is_err());
+    }
+}
diff --git a/KubeOS-Rust/operator/Cargo.toml b/KubeOS-Rust/operator/Cargo.toml
new file mode 100644
index 0000000000000000000000000000000000000000..91cca265469acf57edd1580aae12d121761857ae
--- /dev/null
+++ b/KubeOS-Rust/operator/Cargo.toml
@@ -0,0 +1,47 @@
+[package]
+description = "KubeOS os-operator"
+edition = "2021"
+license = "MulanPSL-2.0"
+name = "operator"
+version = "1.0.6"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[[bin]]
+name = "operator"
+path = "src/main.rs"
+
+[dependencies]
+anyhow = "1.0.44"
+async-trait = "0.1"
+cli = { version = "1.0.6", path = "../cli" }
+env_logger = "0.9.0"
+futures = "0.3.17"
+h2 = "=0.3.16"
+k8s-openapi = { version = "0.13.1", features = ["v1_22"] }
+kube = { version = "0.66.0", features = ["derive", "runtime"] }
+log = "=0.4.15"
+manager = { version = "1.0.6", path = "../manager" }
+regex = "=1.7.3"
+reqwest = { version = "=0.12.2", default-features = false, features = [
+    "json",
+] }
+schemars = "=0.8.10"
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = "1.0.68"
+socket2 = "=0.4.9"
+thiserror = "1.0.29"
+thread_local = "=1.1.4"
+tokio = { version = "=1.28.0", default-features = false, features = [
+    "macros",
+    "rt-multi-thread",
+] }
+tokio-retry = "0.3"
+
+[dev-dependencies]
+assert-json-diff = "2.0.2"
+http = "0.2.9"
+hyper = "0.14.25"
+tower-test = "0.4.0"
+mockall = { version = "=0.11.3" }
+regex = "1"
diff --git a/KubeOS-Rust/operator/src/controller/apiclient.rs b/KubeOS-Rust/operator/src/controller/apiclient.rs
new file mode 100644
index 0000000000000000000000000000000000000000..3642b475dca5ec14f12c3c112df671c4880e7f03
--- /dev/null
+++ b/KubeOS-Rust/operator/src/controller/apiclient.rs
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+
+use anyhow::Result;
+use apiclient_error::Error;
+use async_trait::async_trait;
+use kube::{
+    api::{Api, Patch, PatchParams},
+    Client,
+};
+use serde::{Deserialize, Serialize};
+use super::{
+    crd::{OSInstance, OSInstanceSpec, OSInstanceStatus},
+    values::{NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND},
+};
+
+#[derive(Debug, Serialize, Deserialize)]
+struct OSInstanceSpecPatch {
+    #[serde(rename = "apiVersion")]
+    api_version: String,
+    kind: String,
+    spec: OSInstanceSpec,
+}
+
+impl Default for OSInstanceSpecPatch {
+    fn default() -> Self {
+        OSInstanceSpecPatch {
+            api_version: OSINSTANCE_API_VERSION.to_string(),
+            kind: OSINSTANCE_KIND.to_string(),
+            spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None },
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct OSInstanceStatusPatch {
+    #[serde(rename = "apiVersion")]
+    api_version: String,
+    kind: String,
+    status: Option<OSInstanceStatus>,
+}
+
+impl Default for OSInstanceStatusPatch {
+    fn default() -> Self {
+        OSInstanceStatusPatch {
+            api_version: OSINSTANCE_API_VERSION.to_string(),
+            kind: OSINSTANCE_KIND.to_string(),
+            status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct ControllerClient {
+    pub client: Client,
+}
+
+impl ControllerClient {
+    pub fn new(client: Client) -> Self {
+        ControllerClient { client }
+    }
+}
+
+#[async_trait]
+pub trait ApplyApi: Clone + Sized + Send + Sync {
+    async fn update_osinstance_spec(
+        &self,
+        node_name: &str,
+        namespace: &str,
+        spec: &OSInstanceSpec,
+    ) -> Result<(), Error>;
+}
+
+#[async_trait]
+impl ApplyApi for ControllerClient {
+
+    async fn update_osinstance_spec(
+        &self,
+        node_name: &str,
+        namespace: &str,
+        spec: &OSInstanceSpec,
+    ) -> Result<(), Error> {
+        let osi_api: Api<OSInstance> = Api::namespaced(self.client.clone(), namespace);
+        let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() };
+        osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?;
+        Ok(())
+    }
+
+}
+pub mod apiclient_error {
+    use thiserror::Error;
+    #[derive(Error, Debug)]
+    pub enum Error {
+        #[error("Kubernetes reported error: {source}")]
+        KubeError {
+            #[from]
+            source: kube::Error,
+        },
+    }
+}
diff --git a/KubeOS-Rust/operator/src/controller/apiserver_mock.rs b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs
new file mode 100644
index 0000000000000000000000000000000000000000..21abd40551d73ee1002e5837a1330fe7ce311671
--- /dev/null
+++ b/KubeOS-Rust/operator/src/controller/apiserver_mock.rs
@@ -0,0 +1,995 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{borrow::BorrowMut, cell::{RefCell, RefMut}, clone, collections::BTreeMap, default};
+use regex::Regex;
+use anyhow::Result;
+use cli::{
+    client::Client,
+    method::{
+        callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod,
+        rollback::RollbackMethod, upgrade::UpgradeMethod,
+    },
+};
+use http::{status, Request, Response};
+use hyper::{body::to_bytes, Body};
+use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod};
+use kube::{
+    api::ObjectMeta,
+    core::{ErrorResponse, ListMeta, ObjectList},
+    Client as KubeClient, Resource, ResourceExt,
+};
+use log::debug;
+use mockall::mock;
+use serde_json::json;
+
+use self::mock_error::Error;
+use super::{
+    crd::{Configs, OSInstanceStatus},
+    values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_CONFIG},
+};
+use crate::controller::{
+    apiclient::{ApplyApi, ControllerClient},
+    crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS},
+    values::{LABEL_MASTER, LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE},
+    OperatorController,
+};
+
+type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
+pub struct ApiServerVerifier(ApiServerHandle);
+
+
+#[derive(Clone, Debug, Default)]
+pub struct K8sResources{
+    pub node_list: Vec<Node>,
+    pub osi_list: Vec<OSInstance>,
+}
+
+pub enum Testcases {
+    Rollback(K8sResources),
+    ConfigNormal(K8sResources),
+    SkipNoOsiNode(K8sResources),
+    ExchangeCurrentAndNext(K8sResources),
+    GetConfigOSInstances(String),
+    CheckUpgrading(String),
+    GetIdleOSInstances(String),
+
+}
+
+pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) {
+    tokio::time::timeout(std::time::Duration::from_secs(5), handle)
+        .await
+        .expect("timeout on mock apiserver")
+        .expect("scenario succeeded")
+}
+
+impl ApiServerVerifier {
+    pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> {
+        tokio::spawn(async move {
+            match cases {
+                Testcases::Rollback(k8s_resc) => {
+                    self.handler_worker_node_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_upgrading_node_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    // 为两个节点上的 osi 升级，重复两次
+                    .handler_osinstance_get_by_node_name(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_replace_node_by_name(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_osinstance_get_by_node_name(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_osinstance_patch_nodestatus_upgrade(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_replace_node_by_name(k8s_resc.clone())
+                    .await
+                },
+                Testcases::ConfigNormal(k8s_resc) => {
+                    self.handler_worker_node_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_config_osi_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_idle_osi_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    // 为两个节点上的 osi 升级，重复两次
+                    .handler_osinstance_patch_spec_config(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_osinstance_patch_spec_config(k8s_resc.clone())
+                    .await
+                },
+                Testcases::SkipNoOsiNode(k8s_resc) => {
+                    self.handler_worker_node_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_upgrading_node_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone())
+                    .await
+                },
+                Testcases::ExchangeCurrentAndNext(k8s_resc) => {
+                    self.handler_worker_node_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_upgrading_node_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_worker_and_no_upgrade_noding_list_get(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    // 为两个节点上的 osi 升级，重复两次
+                    .handler_osinstance_get_by_node_name(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_replace_node_by_name(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_osinstance_get_by_node_name(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_osinstance_patch_nodestatus_exchange(k8s_resc.clone())
+                    .await
+                    .unwrap()
+                    .handler_replace_node_by_name(k8s_resc.clone())
+                    .await
+                },
+                _ => {
+                    Err(Error::ArgumentError)
+                }
+            }
+            .expect("Case completed without errors");
+        })
+    }
+
+    pub fn test_function(self, cases: Testcases) -> tokio::task::JoinHandle<()> {
+        tokio::spawn(async move {
+            match cases {
+                Testcases::GetConfigOSInstances(error) => {
+                    self.handler_config_osi_list_get_error(error)
+                    .await
+                },
+                Testcases::CheckUpgrading(error) => {
+                    self.handler_upgrading_node_list_get_error(error)
+                    .await
+                },
+                Testcases::GetIdleOSInstances(error) => {
+                    self.handler_idle_osi_list_get_error(error)
+                    .await
+                },
+                _ => {
+                    Err(Error::ArgumentError)
+                }
+            }
+            .expect("Case completed without errors");
+        })
+    }
+
+    // 获取所有的 worker 节点，对应于reconcile的第一个 get_nodes 函数
+    async fn handler_worker_node_list_get(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            "/api/v1/nodes?&labelSelector=%21node-role.kubernetes.io%2Fcontrol-plane&limit=0");
+        assert_eq!(request.extensions().get(), Some(&"list"));
+
+        // 将 k8s_resc 中所有的 worker 节点传出
+        let mut nodes = vec![];
+        for node in k8s_resc.node_list.clone() {
+            if !node.labels().contains_key(LABEL_MASTER){
+                nodes.push(node.clone());
+            }
+        }
+        
+        let node_list: ObjectList<Node> = ObjectList {
+            metadata: ListMeta {
+                ..Default::default()
+            },
+            items: nodes,
+        };
+
+        dbg!("handler_worker_node_list_get");
+
+        let response = serde_json::to_vec(&node_list).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+
+        Ok(self)
+    }
+
+    // 获取环境中所有的标签为 upgrading 的节点
+    async fn handler_upgrading_node_list_get(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0");
+        assert_eq!(request.extensions().get(), Some(&"list"));
+
+        // 将 k8s_resc 中标签为正在升级的节点传出
+        let mut nodes = vec![];
+        for node in k8s_resc.node_list.clone() {
+            if node.labels().contains_key(LABEL_UPGRADING){
+                nodes.push(node.clone());
+            }
+        }
+        
+        let node_list: ObjectList<Node> = ObjectList {
+            metadata: ListMeta {
+                ..Default::default()
+            },
+            items: nodes,
+        };
+
+        dbg!("handler_upgrading_node_list_get");
+
+        let response = serde_json::to_vec(&node_list).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+
+        Ok(self)
+    }
+
+    // 获取所有的非 upgrading 的 worker 节点
+    async fn handler_worker_and_no_upgrade_noding_list_get(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+
+        let remove_limit = |input: &str| -> String {
+            let re = Regex::new(r"limit=\d+").unwrap();
+            re.replace_all(input, "").to_string()
+        };
+
+        assert_eq!(
+            remove_limit(request.uri().to_string().as_str()),
+            "/api/v1/nodes?&labelSelector=%21upgrade.openeuler.org%2Fupgrading%2C%21node-role.kubernetes.io%2Fcontrol-plane&");
+        assert_eq!(request.extensions().get(), Some(&"list"));
+
+        // 将 k8s_resc 中所有的非 upgrading 的 worker 节点传出
+        let mut nodes = vec![];
+        for node in k8s_resc.node_list.clone() {
+            if !node.labels().contains_key(LABEL_UPGRADING) && !node.labels().contains_key(LABEL_MASTER){
+                nodes.push(node.clone());
+            }
+        }
+        
+        let node_list: ObjectList<Node> = ObjectList {
+            metadata: ListMeta {
+                ..Default::default()
+            },
+            items: nodes,
+        };
+
+        dbg!("handler_worker_and_no_upgrade_noding_list_get");
+
+        let response = serde_json::to_vec(&node_list).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+
+        Ok(self)
+    }
+
+    async fn handler_osinstance_get_by_node_name(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+
+        // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back
+        let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap();
+        let mut osinstance = OSInstance::set_osi_default("", "");
+        let mut boolean_get_osi = false;
+        for osi in k8s_resc.osi_list.clone() {
+            if osi.name() == req_node_name {
+                boolean_get_osi = true;
+                osinstance = osi.clone();
+                break;
+            }
+        }
+        assert!(boolean_get_osi);
+        
+        println!("handler_osinstance_get_by_node_name: req_node_name: {:?}", req_node_name);
+
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        dbg!("handler_osinstance_get_by_node_name");
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_osinstance_patch_nodestatus_upgrade(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PATCH);
+
+        // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back
+        let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap();
+        let mut osinstance = OSInstance::set_osi_default("", "");
+        let mut boolean_get_osi = false;
+        for osi in k8s_resc.osi_list.clone() {
+            if osi.name() == req_node_name {
+                boolean_get_osi = true;
+                osinstance = osi.clone();
+                break;
+            }
+        }
+        assert!(boolean_get_osi);
+
+        println!("handler_osinstance_patch_nodestatus_upgrade: req_node_name: {:?}", req_node_name);
+
+        let req_body = to_bytes(request.into_body()).await.unwrap();
+        let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime");
+        let spec_json = body_json.get("spec").expect("spec object").clone();
+        let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec");
+        assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string());
+
+        dbg!("handler_osinstance_patch_nodestatus_upgrade");
+        osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string();
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_osinstance_patch_nodestatus_exchange(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PATCH);
+
+        // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back
+        let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap();
+        let mut osinstance = OSInstance::set_osi_default("", "");
+        let mut boolean_get_osi = false;
+        for osi in k8s_resc.osi_list.clone() {
+            if osi.name() == req_node_name {
+                boolean_get_osi = true;
+                osinstance = osi.clone();
+                break;
+            }
+        }
+        assert!(boolean_get_osi);
+
+        println!("handler_osinstance_patch_nodestatus_exchange: req_node_name: {:?}", req_node_name);
+
+        let req_body = to_bytes(request.into_body()).await.unwrap();
+        let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime");
+        let spec_json = body_json.get("spec").expect("spec object").clone();
+        let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec");
+
+        let sysconfigs = Some(
+            Configs{
+                version: Some(String::from("v2")),
+                configs: Some(vec![
+                    Config {
+                        model: Some(String::from("grub.cmdline.next")),
+                        configpath: Some(String::from("")),
+                        contents: Some(vec![
+                            Content {
+                                key: Some(String::from("a")),
+                                value: Some(String::from("1")),
+                                operation: Some(String::from("")),
+                            }
+                        ]),
+                    },
+                    Config {
+                        model: Some(String::from("grub.cmdline.current")),
+                        configpath: Some(String::from("")),
+                        contents: Some(vec![
+                            Content {
+                                key: Some(String::from("b")),
+                                value: Some(String::from("2")),
+                                operation: Some(String::from("")),
+                            }
+                        ]),
+                    },
+                ]),
+            }
+        );
+
+        let upgradeconfigs = Some(
+            Configs{
+                version: Some(String::from("v2")),
+                configs: Some(vec![
+                    Config {
+                        model: Some(String::from("grub.cmdline.current")),
+                        configpath: Some(String::from("")),
+                        contents: Some(vec![
+                            Content {
+                                key: Some(String::from("a")),
+                                value: Some(String::from("1")),
+                                operation: Some(String::from("")),
+                            }
+                        ]),
+                    },
+                    Config {
+                        model: Some(String::from("grub.cmdline.next")),
+                        configpath: Some(String::from("")),
+                        contents: Some(vec![
+                            Content {
+                                key: Some(String::from("b")),
+                                value: Some(String::from("2")),
+                                operation: Some(String::from("")),
+                            }
+                        ]),
+                    },
+                ]),
+            }
+        );
+
+        assert_eq!(spec.sysconfigs.clone(), sysconfigs);
+        assert_eq!(spec.upgradeconfigs.clone(), upgradeconfigs);
+        assert_eq!(spec.nodestatus.clone(), NODE_STATUS_UPGRADE.to_string());
+
+        dbg!("handler_osinstance_patch_nodestatus_exchange");
+        osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string();
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }    
+
+    // 通过节点名称获取对应节点
+    async fn handler_replace_node_by_name(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PUT);
+        
+        // get req_node_name from request uri, and match it from k8s_resc.node_list to get node and send back
+        let req_node_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap();
+        let mut node = Node{..Default::default()};
+        let mut boolean_get_node = false;
+        for node_iter in k8s_resc.node_list.clone() {
+            if node_iter.name() == req_node_name {
+                boolean_get_node = true;
+                node = node_iter.clone();
+                break;
+            }
+        }
+        assert!(boolean_get_node);
+        assert_eq!(request.extensions().get(), Some(&"replace"));
+
+        println!("handler_replace_node_by_name: req_node_name: {:?}", req_node_name);
+
+        let req_body = to_bytes(request.into_body()).await.unwrap();
+        let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime");
+        let metadata_json = body_json.get("metadata").expect("metadata object").clone();
+        let metadata: ObjectMeta = serde_json::from_value(metadata_json).expect("valid metadata");
+        assert!(metadata.labels.unwrap().contains_key(LABEL_UPGRADING));
+
+        // 修改 node 并传出
+        node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string());
+
+        dbg!("handler_replace_node_by_name");
+        let response = serde_json::to_vec(&node).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+
+        Ok(self)
+    }
+
+    // 获取环境中所有的标签为 config 的节点上的 osi
+    async fn handler_config_osi_list_get(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?");
+        assert_eq!(request.extensions().get(), Some(&"list"));
+
+        // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出
+        let mut osis = vec![];
+        for osi in k8s_resc.osi_list.clone() {
+            if osi.spec.nodestatus == NODE_STATUS_CONFIG{
+                osis.push(osi.clone());
+            }
+        }
+        
+        let node_list: ObjectList<OSInstance> = ObjectList {
+            metadata: ListMeta {
+                ..Default::default()
+            },
+            items: osis,
+        };
+
+        dbg!("handler_config_osi_list_get");
+
+        let response = serde_json::to_vec(&node_list).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+
+        Ok(self)
+    }
+
+    // 获取环境中所有的标签为 config 的节点上的 osi
+    async fn handler_idle_osi_list_get(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3");
+        assert_eq!(request.extensions().get(), Some(&"list"));
+
+        // 将 k8s_resc 中 nodestatus 为 config 的 osi 传出
+        let mut osis = vec![];
+        for osi in k8s_resc.osi_list.clone() {
+            if osi.spec.nodestatus == NODE_STATUS_IDLE{
+                osis.push(osi.clone());
+            }
+        }
+        
+        let node_list: ObjectList<OSInstance> = ObjectList {
+            metadata: ListMeta {
+                ..Default::default()
+            },
+            items: osis,
+        };
+
+        dbg!("handler_idle_osi_list_get");
+
+        let response = serde_json::to_vec(&node_list).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+
+        Ok(self)
+    }
+
+    async fn handler_osinstance_patch_spec_config(mut self, k8s_resc: K8sResources) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PATCH);
+
+        // get req_node_name from request uri, and match it from k8s_resc.node_list to get osi and send back
+        let req_osi_name = request.uri().path().split('/').last().unwrap().split('?').next().unwrap();
+        let mut osinstance = OSInstance::set_osi_default("", "");
+        let mut boolean_get_osi = false;
+        for osi in k8s_resc.osi_list.clone() {
+            if osi.name() == req_osi_name {
+                boolean_get_osi = true;
+                osinstance = osi.clone();
+                break;
+            }
+        }
+        assert!(boolean_get_osi);
+
+        println!("handler_osinstance_patch_spec_config: req_osi_name: {:?}", req_osi_name);
+
+        let req_body = to_bytes(request.into_body()).await.unwrap();
+        let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime");
+        let spec_json = body_json.get("spec").expect("spec object").clone();
+        let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec");
+        assert_eq!(spec.nodestatus.clone(), NODE_STATUS_CONFIG.to_string());
+
+        let sysconfig = Some(
+            Configs {
+                version: Some(String::from("v2")),
+                configs: Some(vec![Config {
+                    model: Some(String::from("kernel.sysctl")),
+                    configpath: Some(String::from("")),
+                    contents: 
+                        Some(vec![
+                            Content {
+                                key: Some(String::from("key1")),
+                                value: Some(String::from("a")),
+                                operation: Some(String::from("")),
+                            }, 
+                            Content {
+                                key: Some(String::from("key2")),
+                                value: Some(String::from("b")),
+                                operation: Some(String::from("")),
+                            },
+                        ]),
+                }]),
+            }
+        );
+        assert_eq!(
+            spec.sysconfigs.clone(), 
+            sysconfig
+        );
+
+        dbg!("handler_osinstance_patch_spec_config");
+        osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string();
+        osinstance.spec.sysconfigs = sysconfig;
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_config_osi_list_get_error(mut self, error: String) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?");
+        assert_eq!(request.extensions().get(), Some(&"list"));
+
+        dbg!("handler_config_osi_list_get_error");
+        
+        // 仅序列化 ErrorResponse 部分
+        let error_response = ErrorResponse {
+            status: "Failure".to_string(),
+            message: error,
+            reason: "NotFound".to_string(),
+            code: 404,
+        };
+
+        // 序列化为 JSON
+        let response_body = json!({
+            "status": error_response.status,
+            "message": error_response.message,
+            "reason": error_response.reason,
+            "code": error_response.code,
+        });
+
+        // 构建 HTTP 响应
+        let response = serde_json::to_vec(&response_body).unwrap();
+        send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap());
+
+        Ok(self)
+    }
+
+    async fn handler_upgrading_node_list_get_error(mut self, error: String) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            "/api/v1/nodes?&labelSelector=upgrade.openeuler.org%2Fupgrading&limit=0");
+        assert_eq!(request.extensions().get(), Some(&"list"));
+
+        dbg!("handler_upgrading_node_list_get_error");
+
+        // 仅序列化 ErrorResponse 部分
+        let error_response = ErrorResponse {
+            status: "Failure".to_string(),
+            message: error,
+            reason: "Invalid".to_string(),
+            code: 400,
+        };
+
+        // 序列化为 JSON
+        let response_body = json!({
+            "status": error_response.status,
+            "message": error_response.message,
+            "reason": error_response.reason,
+            "code": error_response.code,
+        });
+
+        // 构建 HTTP 响应
+        let response = serde_json::to_vec(&response_body).unwrap();
+        send.send_response(Response::builder().status(400).body(Body::from(response)).unwrap());
+
+        Ok(self)
+    }
+
+    async fn handler_idle_osi_list_get_error(mut self, error: String) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?&limit=3");
+        assert_eq!(request.extensions().get(), Some(&"list"));
+
+        dbg!("handler_idle_osi_list_get_error");
+        
+        // 仅序列化 ErrorResponse 部分
+        let error_response = ErrorResponse {
+            status: "Failure".to_string(),
+            message: error,
+            reason: "NotFound".to_string(),
+            code: 404,
+        };
+
+        // 序列化为 JSON
+        let response_body = json!({
+            "status": error_response.status,
+            "message": error_response.message,
+            "reason": error_response.reason,
+            "code": error_response.code,
+        });
+
+        // 构建 HTTP 响应
+        let response = serde_json::to_vec(&response_body).unwrap();
+        send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap());
+
+        Ok(self)
+    }
+    
+}
+
+pub mod mock_error {
+    use thiserror::Error;
+
+    #[derive(Error, Debug)]
+    pub enum Error {
+        #[error("Kubernetes reported error: {source}")]
+        KubeError {
+            #[from]
+            source: kube::Error,
+        },
+
+        #[error("Parameters other than expected were entered")]
+        ArgumentError,
+    }
+}
+
+
+impl<T: ApplyApi> OperatorController<T> {
+    pub fn test() -> (OperatorController<ControllerClient>, ApiServerVerifier) {
+        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
+        let mock_k8s_client = KubeClient::new(mock_service, "default");
+        let mock_api_client = ControllerClient::new(mock_k8s_client.clone());
+        let operator_controller: OperatorController<ControllerClient> =
+            OperatorController::new(mock_k8s_client, mock_api_client);
+        (operator_controller, ApiServerVerifier(handle))
+    }
+}
+
+impl OSInstance {
+    pub fn set_osi_default(node_name: &str, namespace: &str) -> Self {
+        // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1
+        let mut labels = BTreeMap::new();
+        labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string());
+        OSInstance {
+            metadata: ObjectMeta {
+                name: Some(node_name.to_string()),
+                namespace: Some(namespace.to_string()),
+                labels: Some(labels),
+                ..ObjectMeta::default()
+            },
+            spec: OSInstanceSpec {
+                nodestatus: NODE_STATUS_IDLE.to_string(),
+                sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+                upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+            },
+            status: Some(OSInstanceStatus {
+                sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+                upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+            }),
+        }
+    }
+}
+
+impl OS {
+    pub fn set_os_default() -> Self {
+        let mut os = OS::new("test", OSSpec::default());
+        os.meta_mut().namespace = Some("default".into());
+        os
+    }
+
+    pub fn set_os_rollback_osversion_v1_upgradecon_v1() -> Self {
+        let mut os = OS::set_os_default();
+        os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string();
+        os
+    }
+
+    pub fn set_os_syscon_v2_opstype_config() -> Self {
+        let mut os = OS::set_os_default();
+        os.spec.opstype = OPERATION_TYPE_CONFIG.to_string();
+        os.spec.sysconfigs = Some(
+            Configs {
+                version: Some(String::from("v2")),
+                configs: Some(vec![Config {
+                    model: Some(String::from("kernel.sysctl")),
+                    configpath: Some(String::from("")),
+                    contents: Some(vec![
+                        Content {
+                            key: Some(String::from("key1")),
+                            value: Some(String::from("a")),
+                            operation: Some(String::from("")),
+                        }, 
+                        Content {
+                            key: Some(String::from("key2")),
+                            value: Some(String::from("b")),
+                            operation: Some(String::from("")),
+                        },
+                    ]),
+                }]),
+            }
+        );
+        os
+    }
+
+    pub fn set_os_skip_osversion_v2_upgradecon_v1() -> Self {
+        let mut os = OS::set_os_default();
+        os.spec.osversion = String::from("KubeOS v2");
+        os
+    }
+
+    pub fn set_os_exchange_current_and_next() -> Self {
+        let mut os = OS::set_os_default();
+        os.spec.osversion = String::from("KubeOS v2");
+        let sysconfigs = Some(
+            Configs{
+                version: Some(String::from("v2")),
+                configs: Some(vec![
+                    Config {
+                        model: Some(String::from("grub.cmdline.current")),
+                        configpath: Some(String::from("")),
+                        contents: Some(vec![
+                            Content {
+                                key: Some(String::from("a")),
+                                value: Some(String::from("1")),
+                                operation: Some(String::from("")),
+                            }
+                        ]),
+                    },
+                    Config {
+                        model: Some(String::from("grub.cmdline.next")),
+                        configpath: Some(String::from("")),
+                        contents: Some(vec![
+                            Content {
+                                key: Some(String::from("b")),
+                                value: Some(String::from("2")),
+                                operation: Some(String::from("")),
+                            }
+                        ]),
+                    },
+                ]),
+            }
+        );
+        os.spec.sysconfigs = sysconfigs.clone();
+        os.spec.upgradeconfigs = sysconfigs.clone();
+
+        os
+    }
+
+}
+
+impl K8sResources {
+    pub fn set_rollback_nodes_v2_and_osi_v1() -> Self {
+        // 创建 node1 和 node2
+        let node1 = Node {
+            metadata: ObjectMeta {
+                name: Some("openeuler-node1".into()),
+                labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])),
+                ..Default::default()
+            },
+            spec: None,
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo {
+                    os_image: "KubeOS v2".into(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        let node2 = Node {
+            metadata: ObjectMeta {
+                name: Some("openeuler-node2".into()),
+                labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])),
+                ..Default::default()
+            },
+            spec: None,
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo {
+                    os_image: "KubeOS v2".into(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default");
+        let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default");
+        
+        let node_list = Vec::from([node1, node2]);
+        let osi_list = Vec::from([osi1, osi2]);
+
+        K8sResources{
+            node_list,
+            osi_list
+        }
+    }
+
+    pub fn set_nodes_v1_and_osi_v1() -> Self {
+        // 创建 node1 和 node2
+        let node1 = Node {
+            metadata: ObjectMeta {
+                name: Some("openeuler-node1".into()),
+                labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])),
+                ..Default::default()
+            },
+            spec: None,
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo {
+                    os_image: "KubeOS v1".into(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        let node2 = Node {
+            metadata: ObjectMeta {
+                name: Some("openeuler-node2".into()),
+                labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])),
+                ..Default::default()
+            },
+            spec: None,
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo {
+                    os_image: "KubeOS v1".into(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        let osi1 = OSInstance::set_osi_default(&node1.name().clone(), "default");
+        let osi2 = OSInstance::set_osi_default(&node2.name().clone(), "default");
+        
+        let node_list = Vec::from([node1, node2]);
+        let osi_list = Vec::from([osi1, osi2]);
+
+        K8sResources{
+            node_list,
+            osi_list
+        }
+    }
+
+    pub fn set_skip_nodes_and_osi() -> Self {
+        // 创建 node1 并且不设置 osi
+        let node1 = Node {
+            metadata: ObjectMeta {
+                name: Some("openeuler-node1".into()),
+                labels: Some(BTreeMap::from([("beta.kubernetes.io/os".into(), "linux".into())])),
+                ..Default::default()
+            },
+            spec: None,
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo {
+                    os_image: "KubeOS v1".into(),
+                    ..Default::default()
+                }),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+        
+        let node_list = Vec::from([node1]);
+        let osi_list = Vec::new();
+
+        K8sResources{
+            node_list,
+            osi_list
+        }
+    }
+
+}
+
+impl Default for OSSpec {
+    fn default() -> Self {
+        OSSpec {
+            osversion: String::from("KubeOS v1"),
+            maxunavailable: 3,
+            checksum: String::from("test"),
+            imagetype: String::from("containerd"),
+            containerimage: String::from("test"),
+            opstype: String::from("upgrade"),
+            evictpodforce: true,
+            imageurl: String::from(""),
+            flagsafe: true,
+            mtls: false,
+            cacert: Some(String::from("")),
+            clientcert: Some(String::from("")),
+            clientkey: Some(String::from("")),
+            sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+            upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+        }
+    }
+}
diff --git a/KubeOS-Rust/operator/src/controller/controller.rs b/KubeOS-Rust/operator/src/controller/controller.rs
new file mode 100644
index 0000000000000000000000000000000000000000..2beb8945ba0203ff6d3b5e5ab61e8b535913b391
--- /dev/null
+++ b/KubeOS-Rust/operator/src/controller/controller.rs
@@ -0,0 +1,696 @@
+/*
+* Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+* KubeOS is licensed under the Mulan PSL v2.
+* You can use this software according to the terms and conditions of the Mulan PSL v2.
+* You may obtain a copy of Mulan PSL v2 at:
+*     http://license.coscl.org.cn/MulanPSL2
+* THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+* IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+* PURPOSE.
+* See the Mulan PSL v2 for more details.
+*/
+
+
+use anyhow::Result;
+use k8s_openapi::api::core::v1::Node;
+use kube::{
+    api::{Api, ListParams, ObjectList, PostParams},
+    core::ErrorResponse,
+    runtime::controller::{Context, ReconcilerAction},
+    Client, ResourceExt,
+};
+use log::{debug, error};
+use reconciler_error::Error;
+
+use crate::controller::values::NODE_STATUS_UPGRADE;
+
+use super::{
+    apiclient::ApplyApi,
+    crd::{Configs, OSInstance, OS},
+    values::{
+        LABEL_MASTER, LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, 
+        NO_REQUEUE, OPERATION_TYPE_CONFIG, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, 
+        REQUEUE_ERROR, REQUEUE_NORMAL, SYS_CONFIG_NAME, UPGRADE_CONFIG_NAME
+    },
+};
+
+#[derive(Clone)]
+pub struct OperatorController<T: ApplyApi> {
+    k8s_client: Client,
+    controller_client: T,
+}
+
+impl<T: ApplyApi> OperatorController<T> {
+    pub fn new(k8s_client: Client, controller_client: T) -> Self {
+        OperatorController {
+            k8s_client,
+            controller_client,
+        }
+    }
+
+    // async fn reconcile(
+    //     &self,
+    //     os: OS,
+    //     ctx: Context<OperatorController<T>>,
+    // ) -> Result<ReconcilerAction, Error> {
+    //     // k8s_client 变量不是 Option 类型，而是直接的 Client 类型，Rust 会确保不为空，不用检查
+        
+    //     // Kube-rs库中的Context类型已经处理了上下文的管理，也不需要初始化空的上下文，proxy组件的rust版本也没初始化
+
+    //     // 调用外部的Reconcile函数
+    //     reconcile(os, ctx).await
+    // }
+
+    // 获取 worker 节点数
+    async fn get_and_update_os(&self, _namespace: &str) -> Result<i64, Error> {
+
+        // 创建一个筛选标签的 String 数组，只找 worker 节点
+        let reqs = vec![
+            format!("!{}", LABEL_MASTER),
+        ];
+
+        // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量
+        let nodes_items = self.get_nodes( 0, reqs).await?;
+
+        Ok(nodes_items.items.len() as i64)
+    }
+
+    // 获取 worker 节点，传入 reqs 为 String 数组，表示多个筛选条件
+    async fn get_nodes(&self, limit: i64, reqs: Vec<String>) -> Result<ObjectList<Node>, Error> {
+        
+        let nodes_api: Api<Node> = Api::all(self.k8s_client.clone());
+
+        // 将多个标签筛选器组合成一个字符串，用逗号分隔
+        let label_selector = reqs.join(",");
+
+        // 设置 ListParams
+        let list_params = ListParams::default()
+            .labels(&label_selector)
+            .limit(limit as u32);
+
+        let nodes = match nodes_api.list(&list_params).await {
+            Ok(nodes) => nodes,
+            Err(e) => {
+                log::error!("{:?} unable to list nodes with requirements", e);
+                return Err(Error::KubeClient { source: e });
+            },
+        };
+
+        Ok(nodes)
+    }
+
+    // 获取可以进行升级操作的最大节点数量
+    async fn check_upgrading(&self, _namespace: &str, max_unavailable: i64) -> Result<i64, Error> {
+
+        // 设置筛选标签，选择标签为正在升级的节点
+        let reqs = vec![
+            LABEL_UPGRADING.to_string(),
+        ];
+
+        // 调用getNodes方法获取符合该要求的节点。即worker节点，limit == 0 表示不限制返回数量
+        let nodes_items = self.get_nodes( 0, reqs).await?;
+        
+        Ok(max_unavailable - nodes_items.items.len() as i64)
+    }
+
+    // 为指定数量的节点进行升级操作
+    async fn assign_upgrade(&self, os: &OS, limit: i64, namespace: &str) -> Result<bool, Error> {
+        
+        // 创建筛选标签，只找 worker 节点，并且标签 LabelUpgrading 不存在
+        let reqs = vec![
+            format!("!{}", LABEL_UPGRADING),
+            format!("!{}", LABEL_MASTER),
+        ];
+
+        // 获取符合标签要求的节点
+        let mut nodes_items = self.get_nodes( limit + 1, reqs).await?;
+
+        // 对选定的节点进行升级，返回升级成功的数量和可能的错误
+        let count = self.upgrade_nodes(os, &mut nodes_items, limit, namespace).await?;
+
+        Ok(count >= limit)
+    }
+
+    async fn upgrade_nodes(&self, os: &OS, nodes: &mut ObjectList<Node>, limit: i64, namespace: &str) -> Result<i64, Error> {
+
+        let mut count = 0;
+
+        for node in nodes.iter_mut() {
+            // 如果已经达到升级限制，则退出循环
+            if count >= limit {
+                break
+            }
+
+            let os_version_node = node.status.clone().unwrap().node_info.unwrap().os_image;
+
+            debug!("node name: {}, os_version_node: {}, os_version: {}", node.name(), os_version_node, os.spec.osversion);
+
+            // 检查 os 对象中的操作系统版本是否与节点的操作系统版本不同
+            if os_version_node != os.spec.osversion {
+                
+                // 尝试获取该节点上的 os 实例
+                let osi_api: Api<OSInstance> = Api::namespaced(self.k8s_client.clone(), namespace);
+
+                match osi_api.get(&node.name().clone()).await {
+                    Ok(mut osi) => {
+                        // info!("osinstance is exist {:?}", osi.name());
+
+                        debug!("osinstance is exist: \n {:?} \n", osi);
+
+                        match self.update_node_and_osins(os, node, &mut osi).await {
+                            Ok(_) => {
+                                count += 1;
+                            },
+                            Err(_) => {
+                                continue;
+                            },
+                        }
+                    },
+                    Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => {
+                        debug!("failed to get osInstance {}", &node.name().clone());
+                        
+                        return Err(Error::KubeClient {
+                                    source: kube::Error::Api(ErrorResponse { 
+                                        reason, 
+                                        status: "".to_string(), 
+                                        message: "".to_string(), 
+                                        code: 0 
+                                    })});
+                    },
+                    Err(_) => continue,
+                }
+            }
+
+        }
+
+        Ok(count)
+    }
+
+    // 升级节点以及节点上的 OSinstance
+    async fn update_node_and_osins(&self, os: &OS, node: &mut Node, osinstance: &mut OSInstance, ) -> Result<(), Error> {
+        debug!("start update_node_and_OSins");
+
+        // 检查os实例中的升级配置版本与os对象中的升级配置版本是否匹配。osi 字段未初始化时直接进行拷贝
+        let mut copy_sign = true;
+        if let Some(upgradeconfigs) = osinstance.spec.upgradeconfigs.clone() {
+            if let Some(version) = upgradeconfigs.version {
+                if version == os.spec.upgradeconfigs.clone().unwrap().version.unwrap() {
+                    copy_sign = false;
+                }
+            }
+        }
+        if copy_sign {
+            self.deep_copy_spec_configs(os, osinstance, UPGRADE_CONFIG_NAME.to_string()).await?;
+            assert!(osinstance.spec.upgradeconfigs.is_some());
+        }
+
+        copy_sign = true;
+        // 检查os实例中的系统配置版本与os对象中的系统配置版本是否匹配。osi 字段未初始化时直接进行拷贝
+        if let Some(sysconfigs) = osinstance.spec.sysconfigs.clone() {
+            if let Some(version) = sysconfigs.version {
+                if version == os.spec.sysconfigs.clone().unwrap().version.unwrap() {
+                    copy_sign = false;
+                }
+            }
+        }
+        if copy_sign {
+            self.deep_copy_spec_configs(os, osinstance, SYS_CONFIG_NAME.to_string()).await?;
+            assert!(osinstance.spec.sysconfigs.is_some());
+            if let Some(sysconfigs) = osinstance.spec.sysconfigs.as_mut() {
+                if let Some(configs) = &mut sysconfigs.configs {
+                    for config in configs {
+                        if config.model.clone().unwrap() == "grub.cmdline.current" {
+                            config.model = Some("grub.cmdline.next".to_string());
+                        }
+                        else if config.model.clone().unwrap() == "grub.cmdline.next" {
+                            config.model = Some("grub.cmdline.current".to_string());
+                        }
+                    }
+                }
+            }
+        }
+
+        // 更新os实例中的状态为升级完成状态
+        osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string();
+        
+        // 把对 osinstance 的更改从内存更新到 k8s 集群
+        let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey {
+            resource: String::from("osinstance"),
+            value: String::from("namespace"),
+        })?;
+        self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?;
+
+        node.labels_mut().insert(LABEL_UPGRADING.to_string(), "".to_string());
+
+        // 把对 node 的更改从内存更新到 k8s 集群
+        let node_api: Api<Node> = Api::all(self.k8s_client.clone());
+        node_api.replace(&node.name(), &PostParams::default(), &node).await?;
+
+        Ok(())
+    }
+
+    // 深拷贝
+    async fn deep_copy_spec_configs(&self, os: &OS, os_instance: &mut OSInstance, config_type: String) -> Result<(), Error> {
+        
+        match config_type.as_str() {
+            UPGRADE_CONFIG_NAME =>{
+
+                if let Ok(data) = serde_json::to_vec(&os.spec.upgradeconfigs){
+                    
+                    if let Ok(upgradeconfigs) = serde_json::from_slice(&data) {
+                        os_instance.spec.upgradeconfigs = Some(upgradeconfigs);
+                    }else {
+                        debug!("{} Deserialization failure", config_type);
+                        return Err(Error::Operation { value: "Deserialization".to_string()});
+                    }
+                }
+                else {
+                    debug!("{} Serialization failure", config_type);
+                    return Err(Error::Operation { value: "Serialization".to_string()});
+                }
+
+            },
+            SYS_CONFIG_NAME => {
+
+                if let Ok(data) = serde_json::to_vec(&os.spec.sysconfigs){
+                    
+                    if let Ok(sysconfigs) = serde_json::from_slice(&data) {
+                        os_instance.spec.sysconfigs = Some(sysconfigs);
+                    }else {
+                        debug!("{} Deserialization failure", config_type);
+                        return Err(Error::Operation { value: "Deserialization".to_string()});
+                    }
+                }
+                else {
+                    debug!("{} Serialization failure", config_type);
+                    return Err(Error::Operation { value: "Serialization".to_string()});
+                }
+
+            },
+            _ => {
+                debug!("configType {} cannot be recognized", config_type);
+                return Err(Error::Operation { value: config_type.clone() });
+            },
+        }
+        
+        Ok(())
+    }
+
+    // 获取可以进行配置操作的节点数量，返回的是 instance 的列表
+    async fn check_config(&self, namespace: &str, max_unavailable: i64) -> Result<i64, Error> {
+        let osinstances = self.get_config_osinstances(namespace).await?;
+
+        Ok(max_unavailable - osinstances.len() as i64)
+    }
+
+    // 获取所在节点状态为配置的 osinstance列表
+    async fn get_config_osinstances(&self, namespace: &str) -> Result<Vec<OSInstance>, Error> {
+        let osi_api: Api<OSInstance> = Api::namespaced(self.k8s_client.clone(), namespace);
+
+        // 获取所有 OSInstance 资源
+        let all_osinstances = osi_api.list(&ListParams::default()).await?;
+    
+        // 在客户端进行过滤，节点状态为 NODE_STATUS_CONFIG
+        let osinstances: Vec<OSInstance> = all_osinstances
+            .items
+            .into_iter()
+            .filter(|osi| osi.spec.nodestatus == NODE_STATUS_CONFIG)
+            .collect();
+    
+        debug!("config_osi count = {:?}", osinstances.len());
+    
+        Ok(osinstances)
+    }
+
+    // 为指定数量的节点进行配置操作
+    async fn assign_config(&self, _os: &OS, sysconfigs: Configs, config_version: String, limit: i64, namespace: &str) -> Result<bool, Error> {
+
+        debug!("start assign_config");
+
+        let mut osinstances = self.get_idle_os_instances(namespace, limit + 1).await?;
+
+        let mut count = 0;
+        // 遍历 osi 列表
+        for osi in osinstances.iter_mut() {
+            if count > limit {
+                break;
+            }
+
+            let mut config_sign = true;
+            if let Some(sysconfigs) = osi.spec.sysconfigs.clone() {
+                if let Some(version) = sysconfigs.version {
+                    debug!("node name: {:?}, config_version_node: {:?}, config_version: {:?}", osi.name(), version, config_version);
+                    if version == config_version {
+                        config_sign = false;
+                    }
+                }
+            }
+        
+		    // 如果版本不同或 osi 未初始化，则将新的配置信息更新到实例中，并将节点状态标记为“配置完成”。
+            if config_sign {
+                count += 1;
+                osi.spec.sysconfigs = Some(sysconfigs.clone());
+                osi.spec.nodestatus = NODE_STATUS_CONFIG.to_string();
+                
+                // 把对 osinstance 的更改从内存更新到 k8s 集群
+                let namespace = osi.namespace().ok_or(Error::MissingObjectKey {
+                    resource: String::from("osinstance"),
+                    value: String::from("namespace"),
+                })?;
+                self.controller_client.update_osinstance_spec(&osi.name(), &namespace, &osi.spec).await?;
+            }
+        }
+        
+        Ok(count >= limit)
+    }
+
+    // 获取所在节点状态为空闲的 osinstance列表
+    async fn get_idle_os_instances(&self, namespace: &str, limit: i64) -> Result<Vec<OSInstance>, Error> {
+        
+        let osi_api: Api<OSInstance> = Api::namespaced(self.k8s_client.clone(), namespace);
+
+        // 获取所有 OSInstance 资源
+        let all_osinstances: ObjectList<OSInstance> = osi_api.list(&ListParams::default().limit(limit as u32)).await?;
+
+        // 在客户端进行过滤，节点状态为 NODE_STATUS_IDLE
+        let osinstances: Vec<OSInstance> = all_osinstances
+            .items
+            .into_iter()
+            .filter(|osi| osi.spec.nodestatus == NODE_STATUS_IDLE)
+            .collect();
+    
+        Ok(osinstances)
+    }
+
+
+}
+
+// 调用的函数的具体逻辑需要进一步完成
+pub async fn reconcile<T: ApplyApi>(
+    os: OS,
+    ctx: Context<OperatorController<T>>,
+) -> Result<ReconcilerAction, Error> {
+
+    // 初始化 operator_controller 和 os，从环境变量获取NODE_NAME
+    debug!("start reconcile");
+    let operator_controller = ctx.get_ref();
+    let os_cr: &OS = &os;
+
+    // 从 os_cr 中获取命名空间，如果命名空间不存在则返回错误
+    let namespace: String = os_cr
+        .namespace()
+        .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?;
+
+    debug!("namespace : {:?}", namespace);
+
+    // 获取 worker 节点数
+    let node_num = match operator_controller.get_and_update_os(&namespace).await {
+        Ok(node_num) => node_num,
+        Err(Error::KubeClient { source: kube::Error::Api(ErrorResponse { reason, .. })}) if &reason == "NotFound" => {
+            return Ok(NO_REQUEUE);
+        },
+        Err(_) => return Ok(REQUEUE_ERROR),
+    };
+
+    debug!("node_num : {:?}", node_num);
+
+    let opstype = os_cr.spec.opstype.clone();
+    let ops = opstype.as_str();
+    
+    debug!("opstype: {}", ops);
+
+    match ops {
+        // 如果是升级或者回滚
+        OPERATION_TYPE_UPGRADE | OPERATION_TYPE_ROLLBACK =>{
+            debug!("start upgrade OR rollback");
+
+            // 获取可以进行升级操作的最大节点数量
+            let limit = operator_controller.check_upgrading(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?;
+
+            debug!("limit: {}", limit);
+
+            // 为指定数量的节点进行升级操作
+            let need_requeue = operator_controller.assign_upgrade(os_cr, limit, &namespace).await?;
+            if need_requeue {
+                return Ok(REQUEUE_NORMAL);
+            }
+        },
+        // 配置操作
+        OPERATION_TYPE_CONFIG =>{
+            debug!("start config");
+
+            // 检查待配置的节点数量
+            let limit = operator_controller.check_config(&namespace, os_cr.spec.maxunavailable.min(node_num)).await?;
+
+            debug!("limit: {}", limit);
+
+            // 指派配置任务给节点
+            let sys_configs = os_cr.spec.sysconfigs.clone().unwrap();
+            let version = os_cr.spec.sysconfigs.clone().unwrap().version.unwrap();
+            let need_requeue = operator_controller.assign_config(os_cr, sys_configs, version, limit, &namespace).await?;
+
+            if need_requeue {
+                return Ok(REQUEUE_NORMAL);
+            }
+        },
+        _ =>{
+            log::error!("operation {} cannot be recognized", ops);
+        }
+    }
+    return Ok(REQUEUE_NORMAL);
+}
+
+pub fn error_policy<T: ApplyApi>(
+    error: &Error,
+    _ctx: Context<OperatorController<T>>,
+) -> ReconcilerAction {
+    error!("Reconciliation error: {}", error.to_string());
+    REQUEUE_ERROR
+}
+
+pub mod reconciler_error {
+    use thiserror::Error;
+
+    use crate::controller::{apiclient::apiclient_error};
+
+    #[derive(Error, Debug)]
+    pub enum Error {
+        #[error("Kubernetes reported error: {source}")]
+        KubeClient {
+            #[from]
+            source: kube::Error,
+        },
+
+        #[error("Create/Patch OSInstance reported error: {source}")]
+        ApplyApi {
+            #[from]
+            source: apiclient_error::Error,
+        },
+
+        #[error("Cannot get environment NODE_NAME, error: {source}")]
+        Env {
+            #[from]
+            source: std::env::VarError,
+        },
+
+        #[error("{}.metadata.{} is not exist", resource, value)]
+        MissingObjectKey { resource: String, value: String },
+
+        // #[error("Cannot get {}, {} is None", value, value)]
+        // MissingSubResource { value: String },
+
+        #[error("operation {} cannot be recognized", value)]
+        Operation { value: String },
+
+        // #[error("Expect OS Version is not same with Node OS Version, please upgrade first")]
+        // UpgradeBeforeConfig,
+
+        // #[error("Error when drain node, error reported: {}", value)]
+        // DrainNode { value: String },
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::{borrow::Borrow, cell::RefCell, env};
+
+    use serde::de;
+
+    use super::{error_policy, reconcile, reconciler_error::Error, Context, OSInstance, OperatorController, OS};
+    use crate::controller::{
+        apiserver_mock::{timeout_after_5s, K8sResources, Testcases},
+        ControllerClient,
+    };
+    
+    #[tokio::test]
+    async fn test_rollback() {
+        env::set_var("RUST_LOG", "info");
+        env_logger::init();
+
+        let (test_operator_controller, fakeserver) = OperatorController::<ControllerClient>::test();
+        let os = OS::set_os_rollback_osversion_v1_upgradecon_v1();
+        let context = Context::new(test_operator_controller);
+        let mocksrv = fakeserver
+            .run(Testcases::Rollback(K8sResources::set_rollback_nodes_v2_and_osi_v1()));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_config_normal() {
+        env::set_var("RUST_LOG", "debug");
+        env_logger::init();
+
+        let (test_operator_controller, fakeserver) = OperatorController::<ControllerClient>::test();
+        let os = OS::set_os_syscon_v2_opstype_config();
+        let context = Context::new(test_operator_controller);
+        let mocksrv = fakeserver
+            .run(Testcases::ConfigNormal(K8sResources::set_nodes_v1_and_osi_v1()));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_skip_no_osi_node() {
+        env::set_var("RUST_LOG", "debug");
+        env_logger::init();
+
+        let (test_operator_controller, fakeserver) = OperatorController::<ControllerClient>::test();
+        let os = OS::set_os_skip_osversion_v2_upgradecon_v1();
+        let context = Context::new(test_operator_controller);
+        let mocksrv = fakeserver
+            .run(Testcases::SkipNoOsiNode(K8sResources::set_skip_nodes_and_osi()));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_exchange_current_and_next() {
+        env::set_var("RUST_LOG", "debug");
+        env_logger::init();
+
+        let (test_operator_controller, fakeserver) = OperatorController::<ControllerClient>::test();
+        let os = OS::set_os_exchange_current_and_next();
+        let context = Context::new(test_operator_controller);
+        let mocksrv = fakeserver
+            .run(Testcases::ExchangeCurrentAndNext(K8sResources::set_nodes_v1_and_osi_v1()));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_deep_copy_spec_configs() {
+        env_logger::init();
+
+        let (test_operator_controller, fakeserver) = OperatorController::<ControllerClient>::test();
+        let deep_copy_result = test_operator_controller.clone().deep_copy_spec_configs(&OS::set_os_default(), &mut OSInstance::set_osi_default("", ""), "test".to_string()).await;
+        
+        assert!(deep_copy_result.is_err());
+        
+        if let Err(err) = deep_copy_result {
+            assert_eq!("operation test cannot be recognized".to_string(), err.borrow().to_string());
+        }
+    }
+
+    #[tokio::test]
+    async fn test_get_config_osinstances() {
+        env_logger::init();
+
+        let (test_operator_controller, fakeserver) = OperatorController::<ControllerClient>::test();
+
+        let expected_error = "list error".to_string();
+
+        fakeserver.test_function(Testcases::GetConfigOSInstances(expected_error.clone()));
+
+        // 执行测试
+        let result = test_operator_controller.get_config_osinstances("default").await;
+
+        // 验证返回值
+        assert!(result.is_err());
+        if let Err(err) = result {
+            match err {
+                Error::KubeClient { source } => {
+                    match source {
+                        kube::Error::Api(error_response) => {
+                            assert_eq!(expected_error, error_response.message);
+                        },
+                        _ => {
+                            assert!(false);
+                        }
+                    }
+                }
+                _ => {
+                    assert!(false);
+                }
+            }
+        }
+    }
+
+    #[tokio::test]
+    async fn test_check_upgrading() {
+        env_logger::init();
+
+        let (test_operator_controller, fakeserver) = OperatorController::<ControllerClient>::test();
+
+        fakeserver.test_function(Testcases::CheckUpgrading("label error".to_string()));
+
+        // 执行测试
+        let result = test_operator_controller.check_upgrading("default", 2).await;
+
+        // 验证返回值
+        assert!(result.is_err());
+        if let Err(err) = result {
+            match err {
+                Error::KubeClient { source } => {
+                    match source {
+                        kube::Error::Api(error_response) => {
+                            assert_eq!("label error", error_response.message);
+                        },
+                        _ => {
+                            assert!(false);
+                        }
+                    }
+                }
+                _ => {
+                    assert!(false);
+                }
+            }
+        }
+    }
+
+
+    #[tokio::test]
+    async fn test_get_idle_osinstances() {
+        env_logger::init();
+
+        let (test_operator_controller, fakeserver) = OperatorController::<ControllerClient>::test();
+
+        let expected_error = "list error".to_string();
+
+        fakeserver.test_function(Testcases::GetIdleOSInstances(expected_error.clone()));
+
+        // 执行测试
+        let result = test_operator_controller.get_idle_os_instances("default", 3).await;
+
+        // 验证返回值
+        assert!(result.is_err());
+        if let Err(err) = result {
+            match err {
+                Error::KubeClient { source } => {
+                    match source {
+                        kube::Error::Api(error_response) => {
+                            assert_eq!(expected_error, error_response.message);
+                        },
+                        _ => {
+                            assert!(false);
+                        }
+                    }
+                }
+                _ => {
+                    assert!(false);
+                }
+            }
+        }
+    }
+
+}
diff --git a/KubeOS-Rust/operator/src/controller/crd.rs b/KubeOS-Rust/operator/src/controller/crd.rs
new file mode 100644
index 0000000000000000000000000000000000000000..1fa63035c4eea1c93e4f00402fbfbd331ae10606
--- /dev/null
+++ b/KubeOS-Rust/operator/src/controller/crd.rs
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)]
+#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)]
+pub struct OSSpec {
+    pub osversion: String,
+    pub maxunavailable: i64,
+    pub checksum: String,
+    pub imagetype: String,
+    pub containerimage: String,
+    pub opstype: String,
+    pub evictpodforce: bool,
+    pub imageurl: String,
+    #[serde(rename = "flagSafe")]
+    pub flagsafe: bool,
+    pub mtls: bool,
+    pub cacert: Option<String>,
+    pub clientcert: Option<String>,
+    pub clientkey: Option<String>,
+    #[serde(rename = "sysconfigs")]
+    pub sysconfigs: Option<Configs>,
+    #[serde(rename = "upgradeconfigs")]
+    pub upgradeconfigs: Option<Configs>,
+}
+
+#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)]
+#[kube(
+    group = "upgrade.openeuler.org",
+    version = "v1alpha1",
+    kind = "OSInstance",
+    plural = "osinstances",
+    singular = "osinstance",
+    status = "OSInstanceStatus",
+    namespaced
+)]
+pub struct OSInstanceSpec {
+    pub nodestatus: String,
+    pub sysconfigs: Option<Configs>,
+    pub upgradeconfigs: Option<Configs>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct OSInstanceStatus {
+    pub sysconfigs: Option<Configs>,
+    pub upgradeconfigs: Option<Configs>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct Configs {
+    pub version: Option<String>,
+    pub configs: Option<Vec<Config>>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct Config {
+    pub model: Option<String>,
+    pub configpath: Option<String>,
+    pub contents: Option<Vec<Content>>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct Content {
+    pub key: Option<String>,
+    pub value: Option<String>,
+    pub operation: Option<String>,
+}
diff --git a/KubeOS-Rust/operator/src/controller/mod.rs b/KubeOS-Rust/operator/src/controller/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..468ebfdb711b57c2e5f80f45d236f64c75dcebea
--- /dev/null
+++ b/KubeOS-Rust/operator/src/controller/mod.rs
@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+
+ mod apiclient;
+ #[cfg(test)]
+ mod apiserver_mock;
+ mod controller;
+ mod crd;
+ mod values;
+
+pub use apiclient::ControllerClient;
+pub use controller::{error_policy, reconcile, OperatorController};
+pub use crd::OS;
+ 
\ No newline at end of file
diff --git a/KubeOS-Rust/operator/src/controller/values.rs b/KubeOS-Rust/operator/src/controller/values.rs
new file mode 100644
index 0000000000000000000000000000000000000000..267270a9e3a57f6e2d9595110fc3be6a59ebd761
--- /dev/null
+++ b/KubeOS-Rust/operator/src/controller/values.rs
@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use kube::runtime::controller::ReconcilerAction;
+use tokio::time::Duration;
+
+#[cfg(test)]
+pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node";
+
+pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading";
+
+pub const LABEL_MASTER: &str = "node-role.kubernetes.io/control-plane";
+
+pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1";
+pub const OSINSTANCE_KIND: &str = "OSInstance";
+// pub const OSI_STATUS_NAME: &str = "nodestatus";
+
+pub const UPGRADE_CONFIG_NAME: &str = "UpgradeConfig";
+pub const SYS_CONFIG_NAME: &str = "SysConfig";
+
+pub const NODE_STATUS_IDLE: &str = "idle";
+pub const NODE_STATUS_UPGRADE: &str = "upgrade";
+pub const NODE_STATUS_CONFIG: &str = "config";
+
+pub const OPERATION_TYPE_UPGRADE: &str = "upgrade";
+pub const OPERATION_TYPE_ROLLBACK: &str = "rollback";
+pub const OPERATION_TYPE_CONFIG: &str = "config";
+
+
+pub const NO_REQUEUE: ReconcilerAction = ReconcilerAction { requeue_after: None };
+
+pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) };
+
+pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) };
diff --git a/KubeOS-Rust/operator/src/main.rs b/KubeOS-Rust/operator/src/main.rs
new file mode 100644
index 0000000000000000000000000000000000000000..51c91618516b693bc571a38eb87e12efff548887
--- /dev/null
+++ b/KubeOS-Rust/operator/src/main.rs
@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+ use anyhow::Result;
+ use env_logger::{Builder, Env, Target};
+ use futures::StreamExt;
+ use kube::{
+     api::{Api, ListParams},
+     client::Client,
+     runtime::controller::{Context, Controller},
+ };
+ use log::{error, info};
+ // use std::sync::Arc;
+ use tokio::signal;
+ 
+ mod controller;
+ use controller::{
+     error_policy, reconcile, ControllerClient, OperatorController, OS,
+ };
+ 
+ const OPERATOR_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION");
+ 
+ 
+ #[tokio::main]
+ async fn main() -> Result<()> {
+     // 初始化日志记录器，默认日志级别为info，输出到标准输出
+     Builder::from_env(Env::default().default_filter_or("debug")).target(Target::Stdout).init();
+     
+     // 创建一个默认的Kubernetes客户端
+     let client = Client::try_default().await?;
+ 
+     // 创建一个用于操作OS资源的API对象，自定义的CR
+     let os: Api<OS> = Api::all(client.clone());
+ 
+     // 创建一个控制器客户端
+     let controller_client = ControllerClient::new(client.clone());
+     // let controller_client = Arc::new(ControllerClient::new(client.clone()));
+ 
+     // 创建一个OSReconciler实例
+     let os_reconciler = OperatorController::new(client.clone(), controller_client.clone());
+ 
+     // 记录操作符版本和启动信息
+     info!(
+         "os-operator version is {}, starting operator manager",
+         OPERATOR_VERSION.unwrap_or("Not Found")
+     );
+ 
+     // 创建一个新的控制器并运行，处理reconcile和error_policy，并记录错误信息
+     Controller::new(os, ListParams::default())
+         .run(reconcile, error_policy, Context::new(os_reconciler))
+         .for_each(|res| async move {
+             match res {
+                 Ok(_) => {}
+                 Err(e) => error!("reconcile failed: {}", e.to_string()),
+             }
+         })
+         .await;
+ 
+     // 等待终止信号
+     signal::ctrl_c().await?;
+     info!("os-operator terminated");
+ 
+     Ok(())
+ }
+ 
\ No newline at end of file
diff --git a/KubeOS-Rust/proxy/Cargo.toml b/KubeOS-Rust/proxy/Cargo.toml
new file mode 100644
index 0000000000000000000000000000000000000000..d804ac77111126c1c03ace2649c0c0c29147c6c0
--- /dev/null
+++ b/KubeOS-Rust/proxy/Cargo.toml
@@ -0,0 +1,49 @@
+[package]
+description = "KubeOS os-proxy"
+edition = "2021"
+license = "MulanPSL-2.0"
+name = "proxy"
+version = "1.0.6"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "drain"
+path = "src/drain.rs"
+
+[[bin]]
+name = "proxy"
+path = "src/main.rs"
+
+[dependencies]
+anyhow = "1.0.44"
+async-trait = "0.1"
+cli = { version = "1.0.6", path = "../cli" }
+env_logger = "0.9.0"
+futures = "0.3.17"
+h2 = "=0.3.16"
+k8s-openapi = { version = "0.13.1", features = ["v1_22"] }
+kube = { version = "0.66.0", features = ["derive", "runtime"] }
+log = "=0.4.15"
+manager = { version = "1.0.6", path = "../manager" }
+regex = "=1.7.3"
+reqwest = { version = "=0.12.2", default-features = false, features = [
+    "json",
+] }
+schemars = "=0.8.10"
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = "1.0.68"
+socket2 = "=0.4.9"
+thiserror = "1.0.29"
+thread_local = "=1.1.4"
+tokio = { version = "=1.28.0", default-features = false, features = [
+    "macros",
+    "rt-multi-thread",
+] }
+tokio-retry = "0.3"
+
+[dev-dependencies]
+assert-json-diff = "2.0.2"
+http = "0.2.9"
+hyper = "0.14.25"
+tower-test = "0.4.0"
+mockall = { version = "=0.11.3" }
diff --git a/KubeOS-Rust/proxy/src/controller/agentclient.rs b/KubeOS-Rust/proxy/src/controller/agentclient.rs
new file mode 100644
index 0000000000000000000000000000000000000000..b833f2767806a0ba9c6ff08d904f8c20a4710d9c
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/controller/agentclient.rs
@@ -0,0 +1,153 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{collections::HashMap, path::Path};
+
+use agent_error::Error;
+use cli::{
+    client::Client,
+    method::{
+        callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod,
+        rollback::RollbackMethod, upgrade::UpgradeMethod,
+    },
+};
+use manager::api::{CertsInfo, ConfigureRequest, KeyInfo as AgentKeyInfo, Sysconfig as AgentSysconfig, UpgradeRequest};
+
+pub struct UpgradeInfo {
+    pub version: String,
+    pub image_type: String,
+    pub check_sum: String,
+    pub container_image: String,
+    pub imageurl: String,
+    pub flagsafe: bool,
+    pub mtls: bool,
+    pub cacert: String,
+    pub clientcert: String,
+    pub clientkey: String,
+}
+
+pub struct ConfigInfo {
+    pub configs: Vec<Sysconfig>,
+}
+
+pub struct Sysconfig {
+    pub model: String,
+    pub config_path: String,
+    pub contents: HashMap<String, KeyInfo>,
+}
+
+pub struct KeyInfo {
+    pub value: String,
+    pub operation: String,
+}
+
+pub trait AgentMethod {
+    fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error>;
+    fn upgrade_method(&self) -> Result<(), Error>;
+    fn rollback_method(&self) -> Result<(), Error>;
+    fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error>;
+}
+pub trait AgentCall {
+    fn call_agent<T: RpcMethod + 'static>(&self, client: &Client, method: T) -> Result<(), Error>;
+}
+
+pub struct AgentClient<T: AgentCall> {
+    pub agent_client: Client,
+    pub agent_call_client: T,
+}
+
+impl<T: AgentCall> AgentClient<T> {
+    pub fn new<P: AsRef<Path>>(socket_path: P, agent_call_client: T) -> Self {
+        AgentClient { agent_client: Client::new(socket_path), agent_call_client }
+    }
+}
+
+#[derive(Default)]
+pub struct AgentCallClient {}
+impl AgentCall for AgentCallClient {
+    fn call_agent<T: RpcMethod + 'static>(&self, client: &Client, method: T) -> Result<(), Error> {
+        match method.call(client) {
+            Ok(_resp) => Ok(()),
+            Err(e) => Err(Error::AgentError { source: e }),
+        }
+    }
+}
+
+impl<T: AgentCall> AgentMethod for AgentClient<T> {
+    fn prepare_upgrade_method(&self, upgrade_info: UpgradeInfo) -> Result<(), Error> {
+        let upgrade_request = UpgradeRequest {
+            version: upgrade_info.version,
+            image_type: upgrade_info.image_type,
+            check_sum: upgrade_info.check_sum,
+            container_image: upgrade_info.container_image,
+            image_url: upgrade_info.imageurl,
+            flag_safe: upgrade_info.flagsafe,
+            mtls: upgrade_info.mtls,
+            certs: CertsInfo {
+                ca_cert: upgrade_info.cacert,
+                client_cert: upgrade_info.clientcert,
+                client_key: upgrade_info.clientkey,
+            },
+        };
+        match self.agent_call_client.call_agent(&self.agent_client, PrepareUpgradeMethod::new(upgrade_request)) {
+            Ok(_resp) => Ok(()),
+            Err(e) => Err(e),
+        }
+    }
+
+    fn upgrade_method(&self) -> Result<(), Error> {
+        match self.agent_call_client.call_agent(&self.agent_client, UpgradeMethod::default()) {
+            Ok(_resp) => Ok(()),
+            Err(e) => Err(e),
+        }
+    }
+
+    fn rollback_method(&self) -> Result<(), Error> {
+        match self.agent_call_client.call_agent(&self.agent_client, RollbackMethod::default()) {
+            Ok(_resp) => Ok(()),
+            Err(e) => Err(e),
+        }
+    }
+
+    fn configure_method(&self, config_info: ConfigInfo) -> Result<(), Error> {
+        let mut agent_configs: Vec<AgentSysconfig> = Vec::new();
+        for config in config_info.configs {
+            let mut contents_tmp: HashMap<String, AgentKeyInfo> = HashMap::new();
+            for (key, key_info) in config.contents.iter() {
+                contents_tmp.insert(
+                    key.to_string(),
+                    AgentKeyInfo { value: key_info.value.clone(), operation: key_info.operation.clone() },
+                );
+            }
+            agent_configs.push(AgentSysconfig {
+                model: config.model,
+                config_path: config.config_path,
+                contents: contents_tmp,
+            })
+        }
+        let config_request = ConfigureRequest { configs: agent_configs };
+        match self.agent_call_client.call_agent(&self.agent_client, ConfigureMethod::new(config_request)) {
+            Ok(_resp) => Ok(()),
+            Err(e) => Err(e),
+        }
+    }
+}
+
+pub mod agent_error {
+    use thiserror::Error;
+
+    #[derive(Error, Debug)]
+    pub enum Error {
+        #[error("{source}")]
+        AgentError { source: anyhow::Error },
+    }
+}
diff --git a/KubeOS-Rust/proxy/src/controller/apiclient.rs b/KubeOS-Rust/proxy/src/controller/apiclient.rs
new file mode 100644
index 0000000000000000000000000000000000000000..85e839f0eaa44cb40db6cf097c8113983c871c58
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/controller/apiclient.rs
@@ -0,0 +1,147 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::collections::BTreeMap;
+
+use anyhow::Result;
+use apiclient_error::Error;
+use async_trait::async_trait;
+use kube::{
+    api::{Api, ObjectMeta, Patch, PatchParams, PostParams},
+    Client,
+};
+use serde::{Deserialize, Serialize};
+
+use super::{
+    crd::{OSInstance, OSInstanceSpec, OSInstanceStatus},
+    values::{LABEL_OSINSTANCE, NODE_STATUS_IDLE, OSINSTANCE_API_VERSION, OSINSTANCE_KIND},
+};
+
+#[derive(Debug, Serialize, Deserialize)]
+struct OSInstanceSpecPatch {
+    #[serde(rename = "apiVersion")]
+    api_version: String,
+    kind: String,
+    spec: OSInstanceSpec,
+}
+
+impl Default for OSInstanceSpecPatch {
+    fn default() -> Self {
+        OSInstanceSpecPatch {
+            api_version: OSINSTANCE_API_VERSION.to_string(),
+            kind: OSINSTANCE_KIND.to_string(),
+            spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None ,namespacedname:None},
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct OSInstanceStatusPatch {
+    #[serde(rename = "apiVersion")]
+    api_version: String,
+    kind: String,
+    status: Option<OSInstanceStatus>,
+}
+
+impl Default for OSInstanceStatusPatch {
+    fn default() -> Self {
+        OSInstanceStatusPatch {
+            api_version: OSINSTANCE_API_VERSION.to_string(),
+            kind: OSINSTANCE_KIND.to_string(),
+            status: Some(OSInstanceStatus { sysconfigs: None, upgradeconfigs: None }),
+        }
+    }
+}
+
+#[derive(Clone)]
+pub struct ControllerClient {
+    pub client: Client,
+}
+
+impl ControllerClient {
+    pub fn new(client: Client) -> Self {
+        ControllerClient { client }
+    }
+}
+
+#[async_trait]
+pub trait ApplyApi: Clone + Sized + Send + Sync {
+    async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error>;
+    async fn update_osinstance_spec(
+        &self,
+        node_name: &str,
+        namespace: &str,
+        spec: &OSInstanceSpec,
+    ) -> Result<(), Error>;
+    async fn update_osinstance_status(
+        &self,
+        node_name: &str,
+        namespace: &str,
+        status: &Option<OSInstanceStatus>,
+    ) -> Result<(), Error>;
+}
+
+#[async_trait]
+impl ApplyApi for ControllerClient {
+    async fn create_osinstance(&self, node_name: &str, namespace: &str) -> Result<(), Error> {
+        let mut labels = BTreeMap::new();
+        labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string());
+        let osinstance = OSInstance {
+            metadata: ObjectMeta {
+                name: Some(node_name.to_string()),
+                namespace: Some(namespace.to_string()),
+                labels: Some(labels),
+                ..ObjectMeta::default()
+            },
+            spec: OSInstanceSpec { nodestatus: NODE_STATUS_IDLE.to_string(), sysconfigs: None, upgradeconfigs: None ,namespacedname: None},
+            status: None,
+        };
+        let osi_api = Api::namespaced(self.client.clone(), namespace);
+        osi_api.create(&PostParams::default(), &osinstance).await?;
+        Ok(())
+    }
+
+    async fn update_osinstance_spec(
+        &self,
+        node_name: &str,
+        namespace: &str,
+        spec: &OSInstanceSpec,
+    ) -> Result<(), Error> {
+        let osi_api: Api<OSInstance> = Api::namespaced(self.client.clone(), namespace);
+        let osi_spec_patch = OSInstanceSpecPatch { spec: spec.clone(), ..Default::default() };
+        osi_api.patch(node_name, &PatchParams::default(), &Patch::Merge(&osi_spec_patch)).await?;
+        Ok(())
+    }
+
+    async fn update_osinstance_status(
+        &self,
+        node_name: &str,
+        namespace: &str,
+        status: &Option<OSInstanceStatus>,
+    ) -> Result<(), Error> {
+        let osi_api: Api<OSInstance> = Api::namespaced(self.client.clone(), namespace);
+        let osi_status_patch = OSInstanceStatusPatch { status: status.clone(), ..Default::default() };
+        osi_api.patch_status(node_name, &PatchParams::default(), &Patch::Merge(&osi_status_patch)).await?;
+        Ok(())
+    }
+}
+pub mod apiclient_error {
+    use thiserror::Error;
+    #[derive(Error, Debug)]
+    pub enum Error {
+        #[error("Kubernetes reported error: {source}")]
+        KubeError {
+            #[from]
+            source: kube::Error,
+        },
+    }
+}
diff --git a/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs
new file mode 100644
index 0000000000000000000000000000000000000000..679096d3de517488ba99dd53fe5a808d626e0368
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/controller/apiserver_mock.rs
@@ -0,0 +1,686 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::collections::BTreeMap;
+
+use anyhow::Result;
+use cli::{
+    client::Client,
+    method::{
+        callable_method::RpcMethod, configure::ConfigureMethod, prepare_upgrade::PrepareUpgradeMethod,
+        rollback::RollbackMethod, upgrade::UpgradeMethod,
+    },
+};
+use http::{Request, Response};
+use hyper::{body::to_bytes, Body};
+use k8s_openapi::api::core::v1::{Node, NodeSpec, NodeStatus, NodeSystemInfo, Pod};
+use kube::{
+    api::ObjectMeta,
+    core::{ListMeta, ObjectList},
+    Client as KubeClient, Resource, ResourceExt,
+};
+use mockall::mock;
+
+use self::mock_error::Error;
+use super::{
+    agentclient::*,
+    crd::{Configs, NamespacedName, OSInstanceStatus},
+    values::{NODE_STATUS_CONFIG, NODE_STATUS_UPGRADE, OPERATION_TYPE_ROLLBACK},
+};
+use crate::controller::{
+    apiclient::{ApplyApi, ControllerClient},
+    crd::{Config, Content, OSInstance, OSInstanceSpec, OSSpec, OS},
+    values::{LABEL_OSINSTANCE, LABEL_UPGRADING, NODE_STATUS_IDLE},
+    ProxyController,
+};
+
+type ApiServerHandle = tower_test::mock::Handle<Request<Body>, Response<Body>>;
+pub struct ApiServerVerifier(ApiServerHandle);
+
+pub enum Testcases {
+    OSInstanceNotExist(OSInstance),
+    UpgradeNormal(OSInstance),
+    UpgradeUpgradeconfigsVersionMismatch(OSInstance),
+    UpgradeOSInstaceNodestatusConfig(OSInstance),
+    UpgradeOSInstaceNodestatusIdle(OSInstance),
+    ConfigNormal(OSInstance),
+    ConfigVersionMismatchReassign(OSInstance),
+    ConfigVersionMismatchUpdate(OSInstance),
+    Rollback(OSInstance),
+}
+
+pub async fn timeout_after_5s(handle: tokio::task::JoinHandle<()>) {
+    tokio::time::timeout(std::time::Duration::from_secs(5), handle)
+        .await
+        .expect("timeout on mock apiserver")
+        .expect("scenario succeeded")
+}
+
+impl ApiServerVerifier {
+    pub fn run(self, cases: Testcases) -> tokio::task::JoinHandle<()> {
+        tokio::spawn(async move {
+            match cases {
+                Testcases::OSInstanceNotExist(osi) => {
+                    self.handler_osinstance_get_not_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_creation(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_get(osi)
+                        .await
+                },
+                Testcases::UpgradeNormal(osi) => {
+                    self.handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_get_with_label(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_patch_upgradeconfig_v2(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_cordon(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_pod_list_get(osi)
+                        .await
+                },
+                Testcases::UpgradeUpgradeconfigsVersionMismatch(osi) => {
+                    self.handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_get_with_label(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_update_delete_label(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_uncordon(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_patch_nodestatus_idle(osi)
+                        .await
+                },
+                Testcases::UpgradeOSInstaceNodestatusConfig(osi) => {
+                    self.handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_get_with_label(osi.clone())
+                        .await
+                },
+                Testcases::UpgradeOSInstaceNodestatusIdle(osi) => {
+                    self.handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_get_with_label(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_update_delete_label(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_uncordon(osi)
+                        .await
+                },
+                Testcases::ConfigNormal(osi) => {
+                    self.handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_get(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_patch_sysconfig_v2(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_patch_nodestatus_idle(osi)
+                        .await
+                },
+                Testcases::ConfigVersionMismatchReassign(osi) => {
+                    self.handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_get(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_patch_nodestatus_idle(osi)
+                        .await
+                },
+                Testcases::ConfigVersionMismatchUpdate(osi) => {
+                    self.handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_get(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_patch_spec_sysconfig_v2(osi)
+                        .await
+                },
+                Testcases::Rollback(osi) => {
+                    self.handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_get_exist(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_get_with_label(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_osinstance_patch_upgradeconfig_v2(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_cordon(osi.clone())
+                        .await
+                        .unwrap()
+                        .handler_node_pod_list_get(osi)
+                        .await
+                },
+            }
+            .expect("Case completed without errors");
+        })
+    }
+
+    async fn handler_osinstance_get_not_exist(mut self, osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name())
+        );
+        let response_json = serde_json::json!(
+            { "status": "Failure", "message": "osinstances.upgrade.openeuler.org \"openeuler\" not found", "reason": "NotFound", "code": 404 }
+        );
+        dbg!("handler_osinstance_get_not_exist");
+        let response = serde_json::to_vec(&response_json).unwrap();
+        send.send_response(Response::builder().status(404).body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+    async fn handler_osinstance_get_exist(mut self, osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}", osinstance.name())
+        );
+        dbg!("handler_osinstance_get_exist");
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+    async fn handler_osinstance_creation(mut self, osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::POST);
+        assert_eq!(
+            request.uri().to_string(),
+            format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances?")
+        );
+        dbg!("handler_osinstance_creation");
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_osinstance_patch_nodestatus_idle(mut self, mut osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PATCH);
+        assert_eq!(
+            request.uri().to_string(),
+            format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name())
+        );
+
+        let req_body = to_bytes(request.into_body()).await.unwrap();
+        let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime");
+        let spec_json = body_json.get("spec").expect("spec object").clone();
+        let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec");
+        assert_eq!(spec.nodestatus.clone(), NODE_STATUS_IDLE.to_string());
+
+        dbg!("handler_osinstance_patch_nodestatus_idle");
+        osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string();
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_osinstance_patch_upgradeconfig_v2(mut self, mut osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PATCH);
+        assert_eq!(
+            request.uri().to_string(),
+            format!(
+                "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?",
+                osinstance.name()
+            )
+        );
+
+        let req_body = to_bytes(request.into_body()).await.unwrap();
+        let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid document from runtime");
+        let status_json = body_json.get("status").expect("status object").clone();
+        let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status");
+
+        assert_eq!(
+            status.upgradeconfigs.expect("upgradeconfigs is not None").clone(),
+            osinstance.spec.clone().upgradeconfigs.expect("upgradeconfig is not None")
+        );
+
+        osinstance.status.as_mut().unwrap().upgradeconfigs = osinstance.spec.upgradeconfigs.clone();
+
+        dbg!("handler_osinstance_patch_upgradeconfig_v2");
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_osinstance_patch_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PATCH);
+        assert_eq!(
+            request.uri().to_string(),
+            format!(
+                "/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}/status?",
+                osinstance.name()
+            )
+        );
+
+        let req_body = to_bytes(request.into_body()).await.unwrap();
+        let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance");
+        let status_json = body_json.get("status").expect("status object").clone();
+        let status: OSInstanceStatus = serde_json::from_value(status_json).expect("valid status");
+
+        assert_eq!(
+            status.sysconfigs.expect("sysconfigs is not None").clone(),
+            osinstance.spec.clone().sysconfigs.expect("sysconfig is not None")
+        );
+
+        osinstance.status.as_mut().unwrap().sysconfigs = osinstance.spec.sysconfigs.clone();
+
+        dbg!("handler_osinstance_patch_sysconfig_v2");
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_osinstance_patch_spec_sysconfig_v2(mut self, mut osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PATCH);
+        assert_eq!(
+            request.uri().to_string(),
+            format!("/apis/upgrade.openeuler.org/v1alpha1/namespaces/default/osinstances/{}?", osinstance.name())
+        );
+
+        let req_body = to_bytes(request.into_body()).await.unwrap();
+        let body_json: serde_json::Value = serde_json::from_slice(&req_body).expect("valid osinstance");
+        let spec_json = body_json.get("spec").expect("spec object").clone();
+        let spec: OSInstanceSpec = serde_json::from_value(spec_json).expect("valid spec");
+
+        assert_eq!(
+            spec.sysconfigs.expect("upgradeconfigs is not None").clone().version.clone().unwrap(),
+            String::from("v2")
+        );
+
+        osinstance.spec.sysconfigs.as_mut().unwrap().version = Some(String::from("v2"));
+
+        dbg!("handler_osinstance_patch_spec_sysconfig_v2");
+        let response = serde_json::to_vec(&osinstance).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_node_get(mut self, osinstance: OSInstance) -> Result<Self, Error> {
+        // return node with name = openeuler, osimage = KubeOS v1，no upgrade label
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name()));
+        let node = Node {
+            metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() },
+            spec: None,
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }),
+                ..Default::default()
+            }),
+        };
+        assert_eq!(node.name(), String::from("openeuler"));
+        assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1"));
+        dbg!("handler_node_get");
+        let response = serde_json::to_vec(&node.clone()).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_node_get_with_label(mut self, osinstance: OSInstance) -> Result<Self, Error> {
+        // return node with name = openeuler, osimage = KubeOS v1，has upgrade label
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}", osinstance.name()));
+        let mut node = Node {
+            metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() },
+            spec: None,
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }),
+                ..Default::default()
+            }),
+        };
+        let node_labels = node.labels_mut();
+        node_labels.insert(LABEL_UPGRADING.to_string(), "".to_string());
+        assert_eq!(node.name(), String::from("openeuler"));
+        assert_eq!(node.status.as_ref().unwrap().node_info.as_ref().unwrap().os_image, String::from("KubeOS v1"));
+        assert!(node.labels().contains_key(LABEL_UPGRADING));
+        dbg!("handler_node_get_with_label");
+        let response = serde_json::to_vec(&node.clone()).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_node_update_delete_label(mut self, osinstance: OSInstance) -> Result<Self, Error> {
+        // return node with name = openeuler, osimage = KubeOS v1，no upgrade label
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PUT);
+        assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name()));
+        // check request body has upgrade label
+        let node = Node {
+            metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() },
+            spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }),
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }),
+                ..Default::default()
+            }),
+        };
+        dbg!("handler_node_update_delete_label");
+        let response = serde_json::to_vec(&node.clone()).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_node_cordon(mut self, osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PATCH);
+        assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name()));
+        assert_eq!(request.extensions().get(), Some(&"cordon"));
+        let node = Node {
+            metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() },
+            spec: Some(NodeSpec { unschedulable: Some(true), ..Default::default() }),
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }),
+                ..Default::default()
+            }),
+        };
+        dbg!("handler_node_cordon");
+        let response = serde_json::to_vec(&node.clone()).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_node_uncordon(mut self, osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::PATCH);
+        assert_eq!(request.uri().to_string(), format!("/api/v1/nodes/{}?", osinstance.name()));
+        assert_eq!(request.extensions().get(), Some(&"cordon"));
+        let node = Node {
+            metadata: ObjectMeta { name: Some(String::from("openeuler")), ..Default::default() },
+            spec: Some(NodeSpec { unschedulable: Some(false), ..Default::default() }),
+            status: Some(NodeStatus {
+                node_info: Some(NodeSystemInfo { os_image: String::from("KubeOS v1"), ..Default::default() }),
+                ..Default::default()
+            }),
+        };
+        dbg!("handler_node_uncordon");
+        let response = serde_json::to_vec(&node.clone()).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+
+    async fn handler_node_pod_list_get(mut self, osinstance: OSInstance) -> Result<Self, Error> {
+        let (request, send) = self.0.next_request().await.expect("service not called");
+        assert_eq!(request.method(), http::Method::GET);
+        assert_eq!(
+            request.uri().to_string(),
+            format!("/api/v1/pods?&fieldSelector=spec.nodeName%3D{}", osinstance.name())
+        );
+        assert_eq!(request.extensions().get(), Some(&"list"));
+        let pods_list = ObjectList::<Pod> { metadata: ListMeta::default(), items: vec![] };
+        dbg!("handler_node_pod_list_get");
+        let response = serde_json::to_vec(&pods_list).unwrap();
+        send.send_response(Response::builder().body(Body::from(response)).unwrap());
+        Ok(self)
+    }
+}
+
+pub mod mock_error {
+    use thiserror::Error;
+
+    #[derive(Error, Debug)]
+    pub enum Error {
+        #[error("Kubernetes reported error: {source}")]
+        KubeError {
+            #[from]
+            source: kube::Error,
+        },
+    }
+}
+
+mock! {
+    pub AgentCallClient{}
+    impl AgentCall for AgentCallClient{
+        fn call_agent<T: RpcMethod + 'static>(&self, client:&Client, method: T) -> Result<(), agent_error::Error> {
+                Ok(())
+            }
+    }
+
+}
+impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+    pub fn test() -> (ProxyController<ControllerClient, MockAgentCallClient>, ApiServerVerifier) {
+        let (mock_service, handle) = tower_test::mock::pair::<Request<Body>, Response<Body>>();
+        let mock_k8s_client = KubeClient::new(mock_service, "default");
+        let mock_api_client = ControllerClient::new(mock_k8s_client.clone());
+        let mut mock_agent_call_client = MockAgentCallClient::new();
+        mock_agent_call_client.expect_call_agent::<UpgradeMethod>().returning(|_x, _y| Ok(()));
+        mock_agent_call_client.expect_call_agent::<PrepareUpgradeMethod>().returning(|_x, _y| Ok(()));
+        mock_agent_call_client.expect_call_agent::<RollbackMethod>().returning(|_x, _y| Ok(()));
+        mock_agent_call_client.expect_call_agent::<ConfigureMethod>().returning(|_x, _y| Ok(()));
+        let mock_agent_client = AgentClient::new("test", mock_agent_call_client);
+        let proxy_controller: ProxyController<ControllerClient, MockAgentCallClient> =
+            ProxyController::new(mock_k8s_client, mock_api_client, mock_agent_client);
+        (proxy_controller, ApiServerVerifier(handle))
+    }
+}
+
+impl OSInstance {
+    pub fn set_osi_default(node_name: &str, namespace: &str) -> Self {
+        // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1
+        let mut labels = BTreeMap::new();
+        labels.insert(LABEL_OSINSTANCE.to_string(), node_name.to_string());
+        OSInstance {
+            metadata: ObjectMeta {
+                name: Some(node_name.to_string()),
+                namespace: Some(namespace.to_string()),
+                labels: Some(labels),
+                ..ObjectMeta::default()
+            },
+            spec: OSInstanceSpec {
+                nodestatus: NODE_STATUS_IDLE.to_string(),
+                sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+                upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+                namespacedname:Some(NamespacedName{namespace:String::from("default"),name:String::from("test")})
+            },
+            status: Some(OSInstanceStatus {
+                sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+                upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+            }),
+        }
+    }
+
+    pub fn set_osi_nodestatus_upgrade(node_name: &str, namespace: &str) -> Self {
+        // return osinstance with nodestatus = upgrade, upgradeconfig.version=v1, sysconfig.version=v1
+        let mut osinstance = OSInstance::set_osi_default(node_name, namespace);
+        osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string();
+        osinstance
+    }
+
+    pub fn set_osi_nodestatus_config(node_name: &str, namespace: &str) -> Self {
+        // return osinstance with nodestatus = config, upgradeconfig.version=v1, sysconfig.version=v1
+        let mut osinstance = OSInstance::set_osi_default(node_name, namespace);
+        osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string();
+        osinstance
+    }
+
+    pub fn set_osi_upgradecon_v2(node_name: &str, namespace: &str) -> Self {
+        // return osinstance with nodestatus = idle, upgradeconfig.version=v1, sysconfig.version=v1
+        let mut osinstance = OSInstance::set_osi_default(node_name, namespace);
+        osinstance.spec.upgradeconfigs.as_mut().unwrap().version = Some(String::from("v2"));
+        osinstance
+    }
+
+    pub fn set_osi_nodestatus_upgrade_upgradecon_v2(node_name: &str, namespace: &str) -> Self {
+        // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1
+        let mut osinstance = OSInstance::set_osi_default(node_name, namespace);
+        osinstance.spec.nodestatus = NODE_STATUS_UPGRADE.to_string();
+        osinstance.spec.upgradeconfigs = Some(Configs {
+            version: Some(String::from("v2")),
+            configs: Some(vec![Config {
+                model: Some(String::from("kernel.sysctl.persist")),
+                configpath: Some(String::from("/persist/persist.conf")),
+                contents: Some(vec![Content {
+                    key: Some(String::from("kernel.test")),
+                    value: Some(String::from("test")),
+                    operation: Some(String::from("delete")),
+                }]),
+            }]),
+        });
+        osinstance
+    }
+
+    pub fn set_osi_nodestatus_config_syscon_v2(node_name: &str, namespace: &str) -> Self {
+        // return osinstance with nodestatus = upgrade, upgradeconfig.version=v2, sysconfig.version=v1
+        let mut osinstance = OSInstance::set_osi_default(node_name, namespace);
+        osinstance.spec.nodestatus = NODE_STATUS_CONFIG.to_string();
+        osinstance.spec.sysconfigs = Some(Configs {
+            version: Some(String::from("v2")),
+            configs: Some(vec![Config {
+                model: Some(String::from("kernel.sysctl.persist")),
+                configpath: Some(String::from("/persist/persist.conf")),
+                contents: Some(vec![Content {
+                    key: Some(String::from("kernel.test")),
+                    value: Some(String::from("test")),
+                    operation: Some(String::from("delete")),
+                }]),
+            }]),
+        });
+        osinstance
+    }
+}
+
+impl OS {
+    pub fn set_os_default() -> Self {
+        let mut os = OS::new("test", OSSpec::default());
+        os.meta_mut().namespace = Some("default".into());
+        os
+    }
+    
+    pub fn set_os_osversion_v2_opstype_config() -> Self {
+        let mut os = OS::set_os_default();
+        os.spec.osversion = String::from("KubeOS v2");
+        os.spec.opstype = String::from("config");
+        os
+    }
+
+    pub fn set_os_osversion_v2_upgradecon_v2() -> Self {
+        let mut os = OS::set_os_default();
+        os.spec.osversion = String::from("KubeOS v2");
+        os.spec.upgradeconfigs = Some(Configs { version: Some(String::from("v2")), configs: None });
+        os
+    }
+
+    pub fn set_os_syscon_v2_opstype_config() -> Self {
+        let mut os = OS::set_os_default();
+        os.spec.opstype = String::from("config");
+        os.spec.sysconfigs = Some(Configs {
+            version: Some(String::from("v2")),
+            configs: Some(vec![Config {
+                model: Some(String::from("kernel.sysctl.persist")),
+                configpath: Some(String::from("/persist/persist.conf")),
+                contents: Some(vec![Content {
+                    key: Some(String::from("kernel.test")),
+                    value: Some(String::from("test")),
+                    operation: Some(String::from("delete")),
+                }]),
+            }]),
+        });
+        os
+    }
+
+    pub fn set_os_rollback_osversion_v2_upgradecon_v2() -> Self {
+        let mut os = OS::set_os_default();
+        os.spec.osversion = String::from("KubeOS v2");
+        os.spec.opstype = OPERATION_TYPE_ROLLBACK.to_string();
+        os.spec.upgradeconfigs = Some(Configs {
+            version: Some(String::from("v2")),
+            configs: Some(vec![Config {
+                model: Some(String::from("kernel.sysctl.persist")),
+                configpath: Some(String::from("/persist/persist.conf")),
+                contents: Some(vec![Content {
+                    key: Some(String::from("kernel.test")),
+                    value: Some(String::from("test")),
+                    operation: Some(String::from("delete")),
+                }]),
+            }]),
+        });
+        os
+    }
+}
+
+impl Default for OSSpec {
+    fn default() -> Self {
+        OSSpec {
+            osversion: String::from("KubeOS v1"),
+            maxunavailable: 2,
+            checksum: String::from("test"),
+            imagetype: String::from("containerd"),
+            containerimage: String::from("test"),
+            opstype: String::from("upgrade"),
+            evictpodforce: true,
+            imageurl: String::from(""),
+            flagsafe: false,
+            mtls: false,
+            cacert: Some(String::from("")),
+            clientcert: Some(String::from("")),
+            clientkey: Some(String::from("")),
+            sysconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+            upgradeconfigs: Some(Configs { version: Some(String::from("v1")), configs: None }),
+            nodeselector:None,
+            timeinterval:None,
+            timewindow:None,
+            executionmode:None,
+        }
+    }
+}
diff --git a/KubeOS-Rust/proxy/src/controller/controller.rs b/KubeOS-Rust/proxy/src/controller/controller.rs
new file mode 100644
index 0000000000000000000000000000000000000000..787a0e1c9db1fe0f79a77e75220f529d78d45b46
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/controller/controller.rs
@@ -0,0 +1,589 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use std::{collections::HashMap, env};
+
+use anyhow::Result;
+use drain::drain_os;
+use k8s_openapi::api::core::v1::Node;
+use kube::{
+    api::{Api, PostParams},
+    core::ErrorResponse,
+    runtime::controller::{Context, ReconcilerAction},
+    Client, ResourceExt,
+};
+use log::{debug, error, info};
+use reconciler_error::Error;
+
+use super::{
+    agentclient::{AgentCall, AgentClient, AgentMethod, ConfigInfo, KeyInfo, Sysconfig, UpgradeInfo},
+    apiclient::ApplyApi,
+    crd::{Configs, Content, OSInstance, OS},
+    utils::{check_version, get_config_version, ConfigOperation, ConfigType},
+    values::{
+        LABEL_UPGRADING, NODE_STATUS_CONFIG, NODE_STATUS_IDLE, OPERATION_TYPE_ROLLBACK, OPERATION_TYPE_UPGRADE, 
+        OSINSTANCE_NAMESPACE, REQUEUE_ERROR, REQUEUE_NORMAL,LABEL_CONFIGURING, NO_REQUEUE,
+    },
+};
+
+pub async fn reconcile<T: ApplyApi, U: AgentCall>(
+    os: OS,
+    ctx: Context<ProxyController<T, U>>,
+) -> Result<ReconcilerAction, Error> {
+    debug!("start reconcile");
+    let proxy_controller = ctx.get_ref();
+    let os_cr = &os;
+    let node_name = env::var("NODE_NAME")?;
+    let namespace: String = os_cr
+        .namespace()
+        .ok_or(Error::MissingObjectKey { resource: "os".to_string(), value: "namespace".to_string() })?;
+    proxy_controller.check_osi_exisit(&node_name).await?;
+    let controller_res = proxy_controller.get_resources(&node_name).await?;
+    let node = controller_res.node;
+    let mut osinstance = controller_res.osinstance;
+    if let Some(namespacedname) = osinstance.spec.namespacedname.as_ref(){
+        debug!("osinstance correspending os name is {}, namespace is {}",namespacedname.name,namespacedname.namespace);
+        if !(namespacedname.name == os_cr.name() && namespacedname.namespace == namespace){
+            debug!("current os cr name:{}, namespace:{} is not belong to this node",os_cr.name(),namespace);
+            return Ok(NO_REQUEUE)
+        }
+    }else {
+        debug!("osinstance correspending os name is None, not in upgrading or configuring");
+        return Ok(REQUEUE_NORMAL)
+    }
+
+    let node_os_image = &node
+        .status
+        .as_ref()
+        .ok_or(Error::MissingSubResource { value: String::from("node.status") })?
+        .node_info
+        .as_ref()
+        .ok_or(Error::MissingSubResource { value: String::from("node.status.node_info") })?
+        .os_image;
+    debug!("os expected osversion is {}, actual osversion is {}", os_cr.spec.osversion, node_os_image);
+    if check_version(&os_cr.spec.osversion, node_os_image) {
+        match ConfigType::SysConfig.check_config_version(&os, &osinstance) {
+            ConfigOperation::Reassign => {
+                debug!("start reassign");
+                proxy_controller
+                    .refresh_node(
+                        node,
+                        osinstance,
+                        &get_config_version(os_cr.spec.sysconfigs.as_ref()),
+                        ConfigType::SysConfig,
+                    )
+                    .await?;
+                return Ok(REQUEUE_NORMAL);
+            },
+            ConfigOperation::UpdateConfig => {
+                debug!("start update config");
+                osinstance.spec.sysconfigs = os_cr.spec.sysconfigs.clone();
+                proxy_controller
+                    .controller_client
+                    .update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec)
+                    .await?;
+                return Ok(REQUEUE_ERROR);
+            },
+            _ => {},
+        }
+        if node.labels().contains_key(LABEL_UPGRADING) || node.labels().contains_key(LABEL_CONFIGURING) {
+            if osinstance.spec.nodestatus == NODE_STATUS_IDLE {
+                info!(
+                    "node has upgrade/config label , but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin"
+                );
+                proxy_controller
+                    .refresh_node(
+                        node,
+                        osinstance,
+                        &get_config_version(os_cr.spec.upgradeconfigs.as_ref()),
+                        ConfigType::UpgradeConfig,
+                    )
+                    .await?;
+                return Ok(REQUEUE_NORMAL);
+            }
+        proxy_controller.set_config(&mut osinstance, ConfigType::SysConfig).await?;
+        proxy_controller
+            .refresh_node(node, osinstance, &get_config_version(os_cr.spec.sysconfigs.as_ref()), ConfigType::SysConfig)
+            .await?;
+        }
+    } else {
+        if os_cr.spec.opstype == NODE_STATUS_CONFIG {
+            return Err(Error::UpgradeBeforeConfig);
+        }
+        if let ConfigOperation::Reassign = ConfigType::UpgradeConfig.check_config_version(&os, &osinstance) {
+            debug!("start reassign");
+            proxy_controller
+                .refresh_node(
+                    node,
+                    osinstance,
+                    &get_config_version(os_cr.spec.upgradeconfigs.as_ref()),
+                    ConfigType::UpgradeConfig,
+                )
+                .await?;
+            return Ok(REQUEUE_NORMAL);
+        }
+        if node.labels().contains_key(LABEL_UPGRADING) {
+            if osinstance.spec.nodestatus == NODE_STATUS_IDLE {
+                info!(
+                    "node has upgrade label , but osinstance.spec.nodestatus is idle. Operation:refesh node and wait reassgin"
+                );
+                proxy_controller
+                    .refresh_node(
+                        node,
+                        osinstance,
+                        &get_config_version(os_cr.spec.upgradeconfigs.as_ref()),
+                        ConfigType::UpgradeConfig,
+                    )
+                    .await?;
+                return Ok(REQUEUE_NORMAL);
+            }
+            proxy_controller.set_config(&mut osinstance, ConfigType::UpgradeConfig).await?;
+            proxy_controller.upgrade_node(os_cr, &node).await?;
+        }
+    }
+    Ok(REQUEUE_NORMAL)
+}
+
+pub fn error_policy<T: ApplyApi, U: AgentCall>(
+    error: &Error,
+    _ctx: Context<ProxyController<T, U>>,
+) -> ReconcilerAction {
+    error!("Reconciliation error:{}", error.to_string());
+    REQUEUE_ERROR
+}
+
+struct ControllerResources {
+    osinstance: OSInstance,
+    node: Node,
+}
+pub struct ProxyController<T: ApplyApi, U: AgentCall> {
+    k8s_client: Client,
+    controller_client: T,
+    agent_client: AgentClient<U>,
+}
+
+impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+    pub fn new(k8s_client: Client, controller_client: T, agent_client: AgentClient<U>) -> Self {
+        ProxyController { k8s_client, controller_client, agent_client }
+    }
+}
+
+impl<T: ApplyApi, U: AgentCall> ProxyController<T, U> {
+    async fn check_osi_exisit(&self, node_name: &str) -> Result<(), Error> {
+        let osi_api: Api<OSInstance> = Api::namespaced(self.k8s_client.clone(), OSINSTANCE_NAMESPACE);
+        match osi_api.get(node_name).await {
+            Ok(osi) => {
+                debug!("osinstance is exist {:?}", osi.name());
+                Ok(())
+            },
+            Err(kube::Error::Api(ErrorResponse { reason, .. })) if &reason == "NotFound" => {
+                info!("Create OSInstance {}", node_name);
+                self.controller_client.create_osinstance(node_name, OSINSTANCE_NAMESPACE).await?;
+                Ok(())
+            },
+            Err(err) => Err(Error::KubeClient { source: err }),
+        }
+    }
+
+    async fn get_resources(&self, node_name: &str) -> Result<ControllerResources, Error> {
+        let osi_api: Api<OSInstance> = Api::namespaced(self.k8s_client.clone(), OSINSTANCE_NAMESPACE);
+        let osinstance_cr = osi_api.get(node_name).await?;
+        let node_api: Api<Node> = Api::all(self.k8s_client.clone());
+        let node_cr = node_api.get(node_name).await?;
+        Ok(ControllerResources { osinstance: osinstance_cr, node: node_cr })
+    }
+
+    async fn refresh_node(
+        &self,
+        mut node: Node,
+        osinstance: OSInstance,
+        os_config_version: &str,
+        config_type: ConfigType,
+    ) -> Result<(), Error> {
+        debug!("start refresh_node");
+        let node_api: Api<Node> = Api::all(self.k8s_client.clone());
+        let labels = node.labels_mut();
+        if labels.contains_key(LABEL_UPGRADING) {
+            debug!("delete label {}", LABEL_UPGRADING);
+            labels.remove(LABEL_UPGRADING);
+            node = node_api.replace(&node.name(), &PostParams::default(), &node).await?;
+        }else if labels.contains_key(LABEL_CONFIGURING){
+            debug!("delete label {}", LABEL_CONFIGURING);
+            labels.remove(LABEL_CONFIGURING);
+            node = node_api.replace(&node.name(), &PostParams::default(), &node).await?;
+        }
+        if let Some(node_spec) = &node.spec {
+            if let Some(node_unschedulable) = node_spec.unschedulable {
+                if node_unschedulable {
+                    node_api.uncordon(&node.name()).await?;
+                    info!("Uncordon successfully node{}", node.name());
+                }
+            }
+        }
+        self.update_node_status(osinstance, os_config_version, config_type).await?;
+        Ok(())
+    }
+
+    async fn update_node_status(
+        &self,
+        mut osinstance: OSInstance,
+        os_config_version: &str,
+        config_type: ConfigType,
+    ) -> Result<(), Error> {
+        debug!("start update_node_status");
+        if osinstance.spec.nodestatus == NODE_STATUS_IDLE {
+            return Ok(());
+        }
+        let upgradeconfig_spec_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref());
+        let sysconfig_spec_version = get_config_version(osinstance.spec.sysconfigs.as_ref());
+        let sysconfig_status_version: String;
+        if let Some(osinstance_status) = osinstance.status.as_ref() {
+            sysconfig_status_version = get_config_version(osinstance_status.sysconfigs.as_ref());
+        } else {
+            sysconfig_status_version = get_config_version(None);
+        }
+        if sysconfig_spec_version == sysconfig_status_version
+            || (config_type == ConfigType::SysConfig && os_config_version != sysconfig_spec_version)
+            || (config_type == ConfigType::UpgradeConfig && os_config_version != upgradeconfig_spec_version)
+        {
+            let namespace = osinstance.namespace().ok_or(Error::MissingObjectKey {
+                resource: String::from("osinstance"),
+                value: String::from("namespace"),
+            })?;
+            osinstance.spec.nodestatus = NODE_STATUS_IDLE.to_string();
+            osinstance.spec.namespacedname = None;
+            self.controller_client.update_osinstance_spec(&osinstance.name(), &namespace, &osinstance.spec).await?;
+        }
+        Ok(())
+    }
+
+    async fn update_osi_status(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> {
+        debug!("start update_osi_status");
+        config_type.set_osi_status_config(osinstance);
+        debug!("osinstance status is update to {:?}", osinstance.status);
+        let namespace = &osinstance
+            .namespace()
+            .ok_or(Error::MissingObjectKey { resource: "osinstance".to_string(), value: "namespace".to_string() })?;
+        self.controller_client.update_osinstance_status(&osinstance.name(), namespace, &osinstance.status).await?;
+        Ok(())
+    }
+
+    async fn set_config(&self, osinstance: &mut OSInstance, config_type: ConfigType) -> Result<(), Error> {
+        debug!("start set_config");
+        let config_info = config_type.check_config_start(osinstance);
+        if config_info.need_config {
+            match config_info.configs.and_then(convert_to_agent_config) {
+                Some(agent_configs) => {
+                    match self.agent_client.configure_method(ConfigInfo { configs: agent_configs }) {
+                        Ok(_resp) => {},
+                        Err(e) => {
+                            return Err(Error::Agent { source: e });
+                        },
+                    }
+                },
+                None => {
+                    info!("config is none, No content can be configured.");
+                },
+            };
+            self.update_osi_status(osinstance, config_type).await?;
+        }
+        Ok(())
+    }
+
+    async fn upgrade_node(&self, os_cr: &OS, node: &Node) -> Result<(), Error> {
+        debug!("start upgrade node");
+        match os_cr.spec.opstype.as_str() {
+            OPERATION_TYPE_UPGRADE => {
+                let upgrade_info = UpgradeInfo {
+                    version: os_cr.spec.osversion.clone(),
+                    image_type: os_cr.spec.imagetype.clone(),
+                    check_sum: os_cr.spec.checksum.clone(),
+                    container_image: os_cr.spec.containerimage.clone(),
+                    flagsafe: os_cr.spec.flagsafe,
+                    imageurl: os_cr.spec.imageurl.clone(),
+                    mtls: os_cr.spec.mtls,
+                    cacert: os_cr.spec.cacert.clone().unwrap_or_default(),
+                    clientcert: os_cr.spec.clientcert.clone().unwrap_or_default(),
+                    clientkey: os_cr.spec.clientkey.clone().unwrap_or_default(),
+                };
+
+                match self.agent_client.prepare_upgrade_method(upgrade_info) {
+                    Ok(_resp) => {},
+                    Err(e) => {
+                        return Err(Error::Agent { source: e });
+                    },
+                }
+                self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?;
+                match self.agent_client.upgrade_method() {
+                    Ok(_resp) => {},
+                    Err(e) => {
+                        return Err(Error::Agent { source: e });
+                    },
+                }
+            },
+            OPERATION_TYPE_ROLLBACK => {
+                self.evict_node(&node.name(), os_cr.spec.evictpodforce).await?;
+
+                match self.agent_client.rollback_method() {
+                    Ok(_resp) => {},
+                    Err(e) => {
+                        return Err(Error::Agent { source: e });
+                    },
+                }
+            },
+            _ => {
+                return Err(Error::Operation { value: os_cr.spec.opstype.clone() });
+            },
+        }
+        Ok(())
+    }
+
+    async fn evict_node(&self, node_name: &str, evict_pod_force: bool) -> Result<(), Error> {
+        debug!("start evict_node");
+        let node_api = Api::all(self.k8s_client.clone());
+        node_api.cordon(node_name).await?;
+        info!("Cordon node Successfully{}, start drain nodes", node_name);
+        match self.drain_node(node_name, evict_pod_force).await {
+            Ok(()) => {},
+            Err(e) => {
+                node_api.uncordon(node_name).await?;
+                info!("Drain node {} error, uncordon node successfully", node_name);
+                return Err(e);
+            },
+        }
+        Ok(())
+    }
+
+    async fn drain_node(&self, node_name: &str, force: bool) -> Result<(), Error> {
+        use drain::error::DrainError::*;
+        match drain_os(&self.k8s_client.clone(), node_name, force).await {
+            Err(DeletePodsError { errors, .. }) => Err(Error::DrainNode { value: errors.join("; ") }),
+            _ => Ok(()),
+        }
+    }
+}
+
+fn convert_to_agent_config(configs: Configs) -> Option<Vec<Sysconfig>> {
+    let mut agent_configs: Vec<Sysconfig> = Vec::new();
+    if let Some(config_list) = configs.configs {
+        for config in config_list.into_iter() {
+            match config.contents.and_then(convert_to_config_hashmap) {
+                Some(contents_tmp) => {
+                    let config_tmp = Sysconfig {
+                        model: config.model.unwrap_or_default(),
+                        config_path: config.configpath.unwrap_or_default(),
+                        contents: contents_tmp,
+                    };
+                    agent_configs.push(config_tmp)
+                },
+                None => {
+                    info!(
+                        "model {} which has configpath {} do not has any contents no need to configure",
+                        config.model.unwrap_or_default(),
+                        config.configpath.unwrap_or_default()
+                    );
+                    continue;
+                },
+            };
+        }
+        if agent_configs.is_empty() {
+            info!("no contents in all models, no need to configure");
+            return None;
+        }
+        return Some(agent_configs);
+    }
+    None
+}
+
+fn convert_to_config_hashmap(contents: Vec<Content>) -> Option<HashMap<String, KeyInfo>> {
+    let mut contents_tmp: HashMap<String, KeyInfo> = HashMap::new();
+    for content in contents.into_iter() {
+        let key_info =
+            KeyInfo { value: content.value.unwrap_or_default(), operation: content.operation.unwrap_or_default() };
+        contents_tmp.insert(content.key.unwrap_or_default(), key_info);
+    }
+    Some(contents_tmp)
+}
+
+pub mod reconciler_error {
+    use thiserror::Error;
+
+    use crate::controller::{agentclient::agent_error, apiclient::apiclient_error};
+    #[derive(Error, Debug)]
+    pub enum Error {
+        #[error("Kubernetes reported error: {source}")]
+        KubeClient {
+            #[from]
+            source: kube::Error,
+        },
+
+        #[error("Create/Patch OSInstance reported error: {source}")]
+        ApplyApi {
+            #[from]
+            source: apiclient_error::Error,
+        },
+
+        #[error("Cannot get environment NODE_NAME, error: {source}")]
+        Env {
+            #[from]
+            source: std::env::VarError,
+        },
+
+        #[error("{}.metadata.{} is not exist", resource, value)]
+        MissingObjectKey { resource: String, value: String },
+
+        #[error("Cannot get {}, {} is None", value, value)]
+        MissingSubResource { value: String },
+
+        #[error("operation {} cannot be recognized", value)]
+        Operation { value: String },
+
+        #[error("Expect OS Version is not same with Node OS Version, please upgrade first")]
+        UpgradeBeforeConfig,
+
+        #[error("os-agent reported error:{source}")]
+        Agent { source: agent_error::Error },
+
+        #[error("Error when drain node, error reported: {}", value)]
+        DrainNode { value: String },
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::env;
+
+    use super::{error_policy, reconcile, Context, OSInstance, ProxyController, OS};
+    use crate::controller::{
+        apiserver_mock::{timeout_after_5s, MockAgentCallClient, Testcases},
+        ControllerClient,
+    };
+
+    #[tokio::test]
+    async fn test_create_osinstance_with_no_upgrade_or_configuration() {
+        let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+        env::set_var("NODE_NAME", "openeuler");
+        let os = OS::set_os_default();
+        let context = Context::new(test_proxy_controller);
+        let mocksrv =
+            fakeserver.run(Testcases::OSInstanceNotExist(OSInstance::set_osi_default("openeuler", "default")));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+    #[tokio::test]
+    async fn test_upgrade_normal() {
+        let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+        env::set_var("NODE_NAME", "openeuler");
+        let os = OS::set_os_osversion_v2_upgradecon_v2();
+        let context = Context::new(test_proxy_controller);
+        let mocksrv = fakeserver.run(Testcases::UpgradeNormal(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2(
+            "openeuler",
+            "default",
+        )));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_diff_osversion_opstype_config() {
+        let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+        env::set_var("NODE_NAME", "openeuler");
+        let os = OS::set_os_osversion_v2_opstype_config();
+        let context = Context::new(test_proxy_controller);
+        let mocksrv = fakeserver.run(Testcases::UpgradeOSInstaceNodestatusConfig(
+            OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default"),
+        ));
+        let res = reconcile(os, context.clone()).await;
+        timeout_after_5s(mocksrv).await;
+        assert!(res.is_err(), "upgrade fails due to opstype=config");
+        let err = res.unwrap_err();
+        assert!(err.to_string().contains("Expect OS Version is not same with Node OS Version, please upgrade first"));
+        error_policy(&err, context);
+    }
+
+    #[tokio::test]
+    async fn test_upgradeconfigs_version_mismatch() {
+        let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+        env::set_var("NODE_NAME", "openeuler");
+        let os = OS::set_os_osversion_v2_upgradecon_v2();
+        let context = Context::new(test_proxy_controller);
+        let mocksrv = fakeserver.run(Testcases::UpgradeUpgradeconfigsVersionMismatch(
+            OSInstance::set_osi_nodestatus_upgrade("openeuler", "default"),
+        ));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_upgrade_nodestatus_idle() {
+        let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+        env::set_var("NODE_NAME", "openeuler");
+        let os = OS::set_os_osversion_v2_upgradecon_v2();
+        let context = Context::new(test_proxy_controller);
+        let mocksrv = fakeserver
+            .run(Testcases::UpgradeOSInstaceNodestatusIdle(OSInstance::set_osi_upgradecon_v2("openeuler", "default")));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_config_normal() {
+        let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+        env::set_var("NODE_NAME", "openeuler");
+        let os = OS::set_os_syscon_v2_opstype_config();
+        let context = Context::new(test_proxy_controller);
+        let mocksrv = fakeserver
+            .run(Testcases::ConfigNormal(OSInstance::set_osi_nodestatus_config_syscon_v2("openeuler", "default")));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_sysconfig_version_mismatch_reassign() {
+        let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+        env::set_var("NODE_NAME", "openeuler");
+        let os = OS::set_os_syscon_v2_opstype_config();
+        let context = Context::new(test_proxy_controller);
+        let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchReassign(OSInstance::set_osi_nodestatus_config(
+            "openeuler",
+            "default",
+        )));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_sysconfig_version_mismatch_update() {
+        let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+        env::set_var("NODE_NAME", "openeuler");
+        let os = OS::set_os_syscon_v2_opstype_config();
+        let context = Context::new(test_proxy_controller);
+        let mocksrv = fakeserver.run(Testcases::ConfigVersionMismatchUpdate(OSInstance::set_osi_nodestatus_upgrade(
+            "openeuler",
+            "default",
+        )));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+
+    #[tokio::test]
+    async fn test_rollback() {
+        let (test_proxy_controller, fakeserver) = ProxyController::<ControllerClient, MockAgentCallClient>::test();
+        env::set_var("NODE_NAME", "openeuler");
+        let os = OS::set_os_rollback_osversion_v2_upgradecon_v2();
+        let context = Context::new(test_proxy_controller);
+        let mocksrv = fakeserver
+            .run(Testcases::Rollback(OSInstance::set_osi_nodestatus_upgrade_upgradecon_v2("openeuler", "default")));
+        reconcile(os, context.clone()).await.expect("reconciler");
+        timeout_after_5s(mocksrv).await;
+    }
+}
diff --git a/KubeOS-Rust/proxy/src/controller/crd.rs b/KubeOS-Rust/proxy/src/controller/crd.rs
new file mode 100644
index 0000000000000000000000000000000000000000..36e14d59094dc02828e1a3da8b8a994cf2ac9b25
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/controller/crd.rs
@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use kube::CustomResource;
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)]
+#[kube(group = "upgrade.openeuler.org", version = "v1alpha1", kind = "OS", plural = "os", singular = "os", namespaced)]
+pub struct OSSpec {
+    pub osversion: String,
+    pub maxunavailable: i64,
+    pub checksum: String,
+    pub imagetype: String,
+    pub containerimage: String,
+    pub opstype: String,
+    pub evictpodforce: bool,
+    pub imageurl: String,
+    #[serde(rename = "flagSafe")]
+    pub flagsafe: bool,
+    pub mtls: bool,
+    pub cacert: Option<String>,
+    pub clientcert: Option<String>,
+    pub clientkey: Option<String>,
+    pub sysconfigs: Option<Configs>,
+    pub upgradeconfigs: Option<Configs>,
+    pub nodeselector:Option<String>,
+    pub timewindow: Option<TimeWindow>,
+	pub timeinterval: Option<i64>,
+	pub executionmode:Option<String>,
+
+}
+
+#[derive(CustomResource, Debug, Clone, Deserialize, Serialize, JsonSchema)]
+#[kube(
+    group = "upgrade.openeuler.org",
+    version = "v1alpha1",
+    kind = "OSInstance",
+    plural = "osinstances",
+    singular = "osinstance",
+    status = "OSInstanceStatus",
+    namespaced
+)]
+pub struct OSInstanceSpec {
+    pub nodestatus: String,
+    pub namespacedname: Option<NamespacedName>,
+    pub sysconfigs: Option<Configs>,
+    pub upgradeconfigs: Option<Configs>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct OSInstanceStatus {
+    pub sysconfigs: Option<Configs>,
+    pub upgradeconfigs: Option<Configs>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct Configs {
+    pub version: Option<String>,
+    pub configs: Option<Vec<Config>>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct Config {
+    pub model: Option<String>,
+    pub configpath: Option<String>,
+    pub contents: Option<Vec<Content>>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct Content {
+    pub key: Option<String>,
+    pub value: Option<String>,
+    pub operation: Option<String>,
+}
+
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct NamespacedName{
+    pub namespace: String,
+    pub name: String,
+}
+#[derive(Clone, Deserialize, Serialize, Debug, Eq, PartialEq, JsonSchema)]
+pub struct TimeWindow{
+    pub starttime:String,
+    pub endtime:String,
+}
\ No newline at end of file
diff --git a/KubeOS-Rust/proxy/src/controller/mod.rs b/KubeOS-Rust/proxy/src/controller/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..b8a4e6e58556f631289f4737ad747d61d38be7b3
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/controller/mod.rs
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+mod agentclient;
+mod apiclient;
+#[cfg(test)]
+mod apiserver_mock;
+mod controller;
+mod crd;
+mod utils;
+mod values;
+
+pub use agentclient::{AgentCallClient, AgentClient};
+pub use apiclient::ControllerClient;
+pub use controller::{error_policy, reconcile, ProxyController};
+pub use crd::OS;
+pub use values::SOCK_PATH;
diff --git a/KubeOS-Rust/proxy/src/controller/utils.rs b/KubeOS-Rust/proxy/src/controller/utils.rs
new file mode 100644
index 0000000000000000000000000000000000000000..7e7b41d93d65e143809783ead10c61b2607faddb
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/controller/utils.rs
@@ -0,0 +1,154 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use log::{debug, info};
+
+use super::{
+    crd::{Configs, OSInstance, OSInstanceStatus, OS},
+    values::{NODE_STATUS_CONFIG, NODE_STATUS_IDLE, NODE_STATUS_UPGRADE},
+};
+
+#[derive(PartialEq, Clone, Copy)]
+pub enum ConfigType {
+    UpgradeConfig,
+    SysConfig,
+}
+
+pub enum ConfigOperation {
+    DoNothing,
+    Reassign,
+    UpdateConfig,
+}
+
+pub struct ConfigInfo {
+    pub need_config: bool,
+    pub configs: Option<Configs>,
+}
+
+impl ConfigType {
+    pub fn check_config_version(&self, os: &OS, osinstance: &OSInstance) -> ConfigOperation {
+        debug!("start check_config_version");
+        let node_status = &osinstance.spec.nodestatus;
+        if node_status == NODE_STATUS_IDLE {
+            debug!("node status is idle");
+            return ConfigOperation::DoNothing;
+        };
+        match self {
+            ConfigType::UpgradeConfig => {
+                let os_config_version = get_config_version(os.spec.upgradeconfigs.as_ref());
+                let osi_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref());
+                debug!(
+                    "os upgradeconfig version is {}, osinstance spec upragdeconfig version is {}",
+                    os_config_version, osi_config_version
+                );
+                if !check_version(&os_config_version, &osi_config_version) {
+                    info!(
+                        "os.spec.upgradeconfig.version is not equal to oninstance.spec.upragdeconfig.version, operation: reassgin upgrade to get newest upgradeconfigs"
+                    );
+                    return ConfigOperation::Reassign;
+                }
+            },
+            ConfigType::SysConfig => {
+                let os_config_version = get_config_version(os.spec.sysconfigs.as_ref());
+                let osi_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref());
+                debug!(
+                    "os sysconfig version is {},osinstance spec sysconfig version is {}",
+                    os_config_version, osi_config_version
+                );
+                if !check_version(&os_config_version, &osi_config_version) {
+                    if node_status == NODE_STATUS_CONFIG {
+                        info!(
+                            "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: reassgin config to get newest sysconfigs"
+                        );
+                        return ConfigOperation::Reassign;
+                    }
+                    if node_status == NODE_STATUS_UPGRADE {
+                        info!(
+                            "os.spec.sysconfig.version is not equal to oninstance.spec.sysconfig.version, operation: update osinstance.spec.sysconfig and reconcile"
+                        );
+                        return ConfigOperation::UpdateConfig;
+                    }
+                }
+            },
+        };
+        ConfigOperation::DoNothing
+    }
+    pub fn check_config_start(&self, osinstance: &OSInstance) -> ConfigInfo {
+        debug!("start check_config_start");
+        let spec_config_version: String;
+        let status_config_version: String;
+        let configs: Option<Configs>;
+        match self {
+            ConfigType::UpgradeConfig => {
+                spec_config_version = get_config_version(osinstance.spec.upgradeconfigs.as_ref());
+                if let Some(osinstance_status) = osinstance.status.as_ref() {
+                    status_config_version = get_config_version(osinstance_status.upgradeconfigs.as_ref());
+                } else {
+                    status_config_version = get_config_version(None);
+                }
+                configs = osinstance.spec.upgradeconfigs.clone();
+            },
+            ConfigType::SysConfig => {
+                spec_config_version = get_config_version(osinstance.spec.sysconfigs.as_ref());
+                if let Some(osinstance_status) = osinstance.status.as_ref() {
+                    status_config_version = get_config_version(osinstance_status.sysconfigs.as_ref());
+                } else {
+                    status_config_version = get_config_version(None);
+                }
+                configs = osinstance.spec.sysconfigs.clone();
+            },
+        }
+        debug!(
+            "osinstance spec config version is {}, status config version is {}",
+            spec_config_version, status_config_version
+        );
+        if spec_config_version != status_config_version && osinstance.spec.nodestatus != NODE_STATUS_IDLE {
+            return ConfigInfo { need_config: true, configs };
+        }
+        ConfigInfo { need_config: false, configs: None }
+    }
+    pub fn set_osi_status_config(&self, osinstance: &mut OSInstance) {
+        match self {
+            ConfigType::UpgradeConfig => {
+                if let Some(osi_status) = &mut osinstance.status {
+                    osi_status.upgradeconfigs = osinstance.spec.upgradeconfigs.clone();
+                } else {
+                    osinstance.status = Some(OSInstanceStatus {
+                        upgradeconfigs: osinstance.spec.upgradeconfigs.clone(),
+                        sysconfigs: None,
+                    })
+                }
+            },
+            ConfigType::SysConfig => {
+                if let Some(osi_status) = &mut osinstance.status {
+                    osi_status.sysconfigs = osinstance.spec.sysconfigs.clone();
+                } else {
+                    osinstance.status =
+                        Some(OSInstanceStatus { upgradeconfigs: None, sysconfigs: osinstance.spec.sysconfigs.clone() })
+                }
+            },
+        }
+    }
+}
+
+pub fn check_version(version_a: &str, version_b: &str) -> bool {
+    version_a.eq(version_b)
+}
+
+pub fn get_config_version(configs: Option<&Configs>) -> String {
+    if let Some(configs) = configs {
+        if let Some(version) = configs.version.as_ref() {
+            return version.to_string();
+        }
+    };
+    String::from("")
+}
diff --git a/KubeOS-Rust/proxy/src/controller/values.rs b/KubeOS-Rust/proxy/src/controller/values.rs
new file mode 100644
index 0000000000000000000000000000000000000000..2edb3dcb4766c34b27184f5cda81ffdbfc49ad19
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/controller/values.rs
@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use kube::runtime::controller::ReconcilerAction;
+use tokio::time::Duration;
+
+pub const LABEL_OSINSTANCE: &str = "upgrade.openeuler.org/osinstance-node";
+pub const LABEL_UPGRADING: &str = "upgrade.openeuler.org/upgrading";
+pub const LABEL_CONFIGURING: &str = "upgrade.openeuler.org/configuring";
+
+pub const OSINSTANCE_API_VERSION: &str = "upgrade.openeuler.org/v1alpha1";
+pub const OSINSTANCE_KIND: &str = "OSInstance";
+pub const OSINSTANCE_NAMESPACE: &str = "default";
+
+pub const NODE_STATUS_IDLE: &str = "idle";
+pub const NODE_STATUS_UPGRADE: &str = "upgrade";
+pub const NODE_STATUS_CONFIG: &str = "config";
+
+pub const OPERATION_TYPE_UPGRADE: &str = "upgrade";
+pub const OPERATION_TYPE_ROLLBACK: &str = "rollback";
+
+pub const SOCK_PATH: &str = "/run/os-agent/os-agent.sock";
+
+pub const REQUEUE_NORMAL: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(15)) };
+pub const REQUEUE_ERROR: ReconcilerAction = ReconcilerAction { requeue_after: Some(Duration::from_secs(1)) };
+pub const NO_REQUEUE: ReconcilerAction = ReconcilerAction { requeue_after: None };
diff --git a/KubeOS-Rust/proxy/src/drain.rs b/KubeOS-Rust/proxy/src/drain.rs
new file mode 100644
index 0000000000000000000000000000000000000000..64417df3e0c8e6a300497c00087e8b7441a44ecd
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/drain.rs
@@ -0,0 +1,511 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use futures::{stream, StreamExt};
+use k8s_openapi::api::core::v1::{Pod, PodSpec, PodStatus};
+use kube::{
+    api::{EvictParams, ListParams},
+    core::ObjectList,
+    Api, Client, ResourceExt,
+};
+use log::{debug, error, info};
+use reqwest::StatusCode;
+use tokio::time::{sleep, Duration, Instant};
+use tokio_retry::{
+    strategy::{jitter, ExponentialBackoff},
+    RetryIf,
+};
+
+use self::error::{
+    DrainError::{DeletePodsError, GetPodListsError, WaitDeletionError},
+    EvictionError::{EvictionErrorNoRetry, EvictionErrorRetry},
+};
+
+pub const MAX_EVICT_POD_NUM: usize = 5;
+pub const EVERY_EVICTION_RETRY: Duration = Duration::from_secs(5);
+pub const EVERY_DELETION_CHECK: Duration = Duration::from_secs(5);
+pub const TIMEOUT: Duration = Duration::from_secs(u64::MAX);
+pub const RETRY_BASE_DELAY: Duration = Duration::from_millis(100);
+pub const RETRY_MAX_DELAY: Duration = Duration::from_secs(20);
+pub const MAX_RETRIES_TIMES: usize = 10;
+
+pub async fn drain_os(client: &Client, node_name: &str, force: bool) -> Result<(), error::DrainError> {
+    let pods_list = get_pods_deleted(client, node_name, force).await?;
+
+    stream::iter(pods_list)
+        .for_each_concurrent(MAX_EVICT_POD_NUM, move |pod| {
+            let k8s_client = client.clone();
+            async move {
+                if evict_pod(&k8s_client, &pod, force).await.is_ok() {
+                    wait_for_deletion(&k8s_client, &pod).await.ok();
+                }
+            }
+        })
+        .await;
+
+    Ok(())
+}
+
+async fn get_pods_deleted(
+    client: &Client,
+    node_name: &str,
+    force: bool,
+) -> Result<impl Iterator<Item = Pod>, error::DrainError> {
+    let lp = ListParams { field_selector: Some(format!("spec.nodeName={}", node_name)), ..Default::default() };
+    let pods_api: Api<Pod> = Api::all(client.clone());
+    let pods: ObjectList<Pod> = match pods_api.list(&lp).await {
+        Ok(pods @ ObjectList { .. }) => pods,
+        Err(err) => {
+            return Err(GetPodListsError { source: err, node_name: node_name.to_string() });
+        },
+    };
+    let mut filterd_pods_list: Vec<Pod> = Vec::new();
+    let mut filterd_err: Vec<String> = Vec::new();
+    let pod_filter = CombinedFilter::new(force);
+    for pod in pods.into_iter() {
+        let filter_result = pod_filter.filter(&pod);
+        if filter_result.status == PodDeleteStatus::Error {
+            filterd_err.push(filter_result.desc);
+            continue;
+        }
+        if filter_result.result {
+            filterd_pods_list.push(pod);
+        }
+    }
+    if !filterd_err.is_empty() {
+        return Err(DeletePodsError { errors: filterd_err });
+    }
+    Ok(filterd_pods_list.into_iter())
+}
+
+async fn evict_pod(k8s_client: &kube::Client, pod: &Pod, force: bool) -> Result<(), error::EvictionError> {
+    let pod_api: Api<Pod> = get_pod_api_with_namespace(k8s_client, pod);
+
+    let error_handling_strategy =
+        if force { ErrorHandleStrategy::RetryStrategy } else { ErrorHandleStrategy::TolerateStrategy };
+
+    RetryIf::spawn(
+        error_handling_strategy.retry_strategy(),
+        || async {
+            loop {
+                let eviction_result = pod_api.evict(&pod.name_any(), &EvictParams::default()).await;
+
+                match eviction_result {
+                    Ok(_) => {
+                        pod.name();
+                        debug!("Successfully evicted Pod '{}'", pod.name_any());
+                        break;
+                    }
+                    Err(kube::Error::Api(e)) => {
+                        let status_code = StatusCode::from_u16(e.code);
+                        match status_code {
+                            Ok(StatusCode::FORBIDDEN) => {
+                                return Err(EvictionErrorNoRetry {
+                                    source: kube::Error::Api(e.clone()),
+                                    pod_name: pod.name_any(),
+                                });
+                            }
+                            Ok(StatusCode::NOT_FOUND) => {
+                                return Err(EvictionErrorNoRetry {
+                                    source: kube::Error::Api(e.clone()),
+                                    pod_name: pod.name_any(),
+                                });
+                            }
+                            Ok(StatusCode::INTERNAL_SERVER_ERROR) => {
+                                error!(
+                                    "Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to misconfigured PodDisruptionBudgets.",
+                                    pod.name_any(),
+                                    e,
+                                    EVERY_EVICTION_RETRY.as_secs_f64()
+                                );
+                                sleep(EVERY_EVICTION_RETRY).await;
+                                continue;
+                            }
+                            Ok(StatusCode::TOO_MANY_REQUESTS) => {
+                                error!("Evict pod {} reported error: '{}' and will retry in {:.2}s. This error maybe is due to PodDisruptionBugets.",
+                                    pod.name_any(),
+                                    e,
+                                    EVERY_EVICTION_RETRY.as_secs_f64()
+                                );
+                                sleep(EVERY_EVICTION_RETRY).await;
+                                continue;
+                            }
+                            Ok(_) => {
+                                error!(
+                                    "Evict pod {} reported error: '{}'.",
+                                    pod.name_any(),
+                                    e
+                                );
+                                return Err(EvictionErrorRetry {
+                                    source: kube::Error::Api(e.clone()),
+                                    pod_name: pod.name_any(),
+                                });
+                            }
+                            Err(_) => {
+                                error!(
+                                    "Evict pod {} reported error: '{}'.Received invalid response code from Kubernetes API",
+                                    pod.name_any(),
+                                    e
+                                );
+                                return Err(EvictionErrorRetry {
+                                    source: kube::Error::Api(e.clone()),
+                                    pod_name: pod.name_any(),
+                                });
+                            }
+                        }
+                    }
+                    Err(e) => {
+                        error!("Evict pod {} reported error: '{}' and will retry", pod.name_any(),e);
+                        return Err(EvictionErrorRetry {
+                            source: e,
+                            pod_name: pod.name_any(),
+                        });
+                    }
+                }
+            }
+            Ok(())
+        },
+        error_handling_strategy
+    ).await
+}
+
+async fn wait_for_deletion(k8s_client: &kube::Client, pod: &Pod) -> Result<(), error::DrainError> {
+    let start_time = Instant::now();
+
+    let pod_api: Api<Pod> = get_pod_api_with_namespace(k8s_client, pod);
+    let response_error_not_found: u16 = 404;
+    loop {
+        match pod_api.get(&pod.name_any()).await {
+            Ok(p) if p.uid() != pod.uid() => {
+                let name = (&p).name_any();
+                info!("Pod {} deleted.", name);
+                break;
+            },
+            Ok(_) => {
+                info!("Pod '{}' is not yet deleted. Waiting {}s.", pod.name_any(), EVERY_DELETION_CHECK.as_secs_f64());
+            },
+            Err(kube::Error::Api(e)) if e.code == response_error_not_found => {
+                info!("Pod {} is deleted.", pod.name_any());
+                break;
+            },
+            Err(e) => {
+                error!(
+                    "Get pod {} reported error: '{}', whether pod is deleted cannot be determined, waiting {}s.",
+                    pod.name_any(),
+                    e,
+                    EVERY_DELETION_CHECK.as_secs_f64()
+                );
+            },
+        }
+        if start_time.elapsed() > TIMEOUT {
+            return Err(WaitDeletionError { pod_name: pod.name_any(), max_wait: TIMEOUT });
+        } else {
+            sleep(EVERY_DELETION_CHECK).await;
+        }
+    }
+    Ok(())
+}
+
+fn get_pod_api_with_namespace(client: &kube::Client, pod: &Pod) -> Api<Pod> {
+    match pod.metadata.namespace.as_ref() {
+        Some(namespace) => Api::namespaced(client.clone(), namespace),
+        None => Api::default_namespaced(client.clone()),
+    }
+}
+
+trait NameAny {
+    fn name_any(&self) -> String;
+}
+
+impl NameAny for &Pod {
+    fn name_any(&self) -> String {
+        self.metadata.name.clone().or_else(|| self.metadata.generate_name.clone()).unwrap_or_default()
+    }
+}
+trait PodFilter {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult>;
+}
+
+struct FinishedOrFailedFilter {}
+impl PodFilter for FinishedOrFailedFilter {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+        return match pod.status.as_ref() {
+            Some(PodStatus { phase: Some(phase), .. }) if phase == "Failed" || phase == "Succeeded" => {
+                FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay)
+            },
+            _ => FilterResult::create_filter_result(false, "", PodDeleteStatus::Okay),
+        };
+    }
+}
+struct DaemonFilter {
+    finished_or_failed_filter: FinishedOrFailedFilter,
+    force: bool,
+}
+impl PodFilter for DaemonFilter {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+        if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() {
+            return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay);
+        }
+
+        return match pod.metadata.owner_references.as_ref() {
+            Some(owner_references)
+                if owner_references
+                    .iter()
+                    .any(|reference| reference.controller.unwrap_or(false) && reference.kind == "DaemonSet") =>
+            {
+                if self.force {
+                    let description = format!("Ignore Pod '{}': Pod is member of a DaemonSet", pod.name_any());
+                    Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Warning })
+                } else {
+                    let description = format!("Cannot drain Pod '{}': Pod is member of a DaemonSet", pod.name_any());
+                    Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error })
+                }
+            },
+            _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay),
+        };
+    }
+}
+impl DaemonFilter {
+    fn new(force: bool) -> DaemonFilter {
+        DaemonFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force }
+    }
+}
+
+struct MirrorFilter {}
+impl PodFilter for MirrorFilter {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+        return match pod.metadata.annotations.as_ref() {
+            Some(annotations) if annotations.contains_key("kubernetes.io/config.mirror") => {
+                let description = format!("Ignore Pod '{}': Pod is a static Mirror Pod", pod.name_any());
+                FilterResult::create_filter_result(false, &description.to_string(), PodDeleteStatus::Warning)
+            },
+            _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay),
+        };
+    }
+}
+
+struct LocalStorageFilter {
+    finished_or_failed_filter: FinishedOrFailedFilter,
+    force: bool,
+}
+impl PodFilter for LocalStorageFilter {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+        if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() {
+            return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay);
+        }
+
+        return match pod.spec.as_ref() {
+            Some(PodSpec { volumes: Some(volumes), .. }) if volumes.iter().any(|volume| volume.empty_dir.is_some()) => {
+                if self.force {
+                    let description = format!("Force draining Pod '{}': Pod has local storage", pod.name_any());
+                    Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning })
+                } else {
+                    let description = format!("Cannot drain Pod '{}': Pod has local Storage", pod.name_any());
+                    Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error })
+                }
+            },
+            _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay),
+        };
+    }
+}
+impl LocalStorageFilter {
+    fn new(force: bool) -> LocalStorageFilter {
+        LocalStorageFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force }
+    }
+}
+struct UnreplicatedFilter {
+    finished_or_failed_filter: FinishedOrFailedFilter,
+    force: bool,
+}
+impl PodFilter for UnreplicatedFilter {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+        if let FilterResult { result: true, .. } = self.finished_or_failed_filter.filter(pod).as_ref() {
+            return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay);
+        }
+
+        let is_replicated = pod.metadata.owner_references.is_some();
+
+        if is_replicated {
+            return FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay);
+        }
+
+        if !is_replicated && self.force {
+            let description = format!("Force drain Pod '{}': Pod is unreplicated", pod.name_any());
+            Box::new(FilterResult { result: true, desc: description, status: PodDeleteStatus::Warning })
+        } else {
+            let description = format!("Cannot drain Pod '{}': Pod is unreplicated", pod.name_any());
+            Box::new(FilterResult { result: false, desc: description, status: PodDeleteStatus::Error })
+        }
+    }
+}
+impl UnreplicatedFilter {
+    fn new(force: bool) -> UnreplicatedFilter {
+        UnreplicatedFilter { finished_or_failed_filter: FinishedOrFailedFilter {}, force }
+    }
+}
+
+struct DeletedFilter {
+    delete_wait_timeout: Duration,
+}
+impl PodFilter for DeletedFilter {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+        let now = Instant::now().elapsed();
+        return match pod.metadata.deletion_timestamp.as_ref() {
+            Some(time)
+                if time.0.timestamp() != 0
+                    && now - Duration::from_secs(time.0.timestamp() as u64) >= self.delete_wait_timeout =>
+            {
+                FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay)
+            },
+            _ => FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay),
+        };
+    }
+}
+
+struct CombinedFilter {
+    deleted_filter: DeletedFilter,
+    daemon_filter: DaemonFilter,
+    mirror_filter: MirrorFilter,
+    local_storage_filter: LocalStorageFilter,
+    unreplicated_filter: UnreplicatedFilter,
+}
+impl PodFilter for CombinedFilter {
+    fn filter(&self, pod: &Pod) -> Box<FilterResult> {
+        let mut filter_res = self.deleted_filter.filter(pod);
+        if !filter_res.result {
+            info!("{}", filter_res.desc);
+            return Box::new(FilterResult {
+                result: filter_res.result,
+                desc: filter_res.desc.clone(),
+                status: filter_res.status,
+            });
+        }
+        filter_res = self.daemon_filter.filter(pod);
+        if !filter_res.result {
+            info!("{}", filter_res.desc);
+            return Box::new(FilterResult {
+                result: filter_res.result,
+                desc: filter_res.desc.clone(),
+                status: filter_res.status,
+            });
+        }
+        filter_res = self.mirror_filter.filter(pod);
+        if !filter_res.result {
+            info!("{}", filter_res.desc);
+            return Box::new(FilterResult {
+                result: filter_res.result,
+                desc: filter_res.desc.clone(),
+                status: filter_res.status,
+            });
+        }
+        filter_res = self.local_storage_filter.filter(pod);
+        if !filter_res.result {
+            info!("{}", filter_res.desc);
+            return Box::new(FilterResult {
+                result: filter_res.result,
+                desc: filter_res.desc.clone(),
+                status: filter_res.status,
+            });
+        }
+        filter_res = self.unreplicated_filter.filter(pod);
+        if !filter_res.result {
+            info!("{}", filter_res.desc);
+            return Box::new(FilterResult {
+                result: filter_res.result,
+                desc: filter_res.desc.clone(),
+                status: filter_res.status,
+            });
+        }
+
+        FilterResult::create_filter_result(true, "", PodDeleteStatus::Okay)
+    }
+}
+impl CombinedFilter {
+    fn new(force: bool) -> CombinedFilter {
+        CombinedFilter {
+            deleted_filter: DeletedFilter { delete_wait_timeout: TIMEOUT },
+            daemon_filter: DaemonFilter::new(force),
+            mirror_filter: MirrorFilter {},
+            local_storage_filter: LocalStorageFilter::new(force),
+            unreplicated_filter: UnreplicatedFilter::new(force),
+        }
+    }
+}
+
+#[derive(PartialEq, Clone, Copy)]
+enum PodDeleteStatus {
+    Okay,
+    Warning,
+    Error,
+}
+struct FilterResult {
+    result: bool,
+    desc: String,
+    status: PodDeleteStatus,
+}
+impl FilterResult {
+    fn create_filter_result(result: bool, desc: &str, status: PodDeleteStatus) -> Box<FilterResult> {
+        Box::new(FilterResult { result, desc: desc.to_string(), status })
+    }
+}
+
+enum ErrorHandleStrategy {
+    RetryStrategy,
+    TolerateStrategy,
+}
+
+impl ErrorHandleStrategy {
+    fn retry_strategy(&self) -> impl Iterator<Item = Duration> {
+        let backoff =
+            ExponentialBackoff::from_millis(RETRY_BASE_DELAY.as_millis() as u64).max_delay(RETRY_MAX_DELAY).map(jitter);
+
+        match self {
+            Self::TolerateStrategy => backoff.take(0),
+
+            Self::RetryStrategy => backoff.take(MAX_RETRIES_TIMES),
+        }
+    }
+}
+
+impl tokio_retry::Condition<error::EvictionError> for ErrorHandleStrategy {
+    fn should_retry(&mut self, error: &error::EvictionError) -> bool {
+        match self {
+            Self::TolerateStrategy => false,
+            Self::RetryStrategy => matches!(error, error::EvictionError::EvictionErrorRetry { .. }),
+        }
+    }
+}
+
+pub mod error {
+    use thiserror::Error;
+    use tokio::time::Duration;
+
+    #[derive(Debug, Error)]
+    pub enum DrainError {
+        #[error("Get node {} pods list error reported: {}", node_name, source)]
+        GetPodListsError { source: kube::Error, node_name: String },
+
+        #[error("Pod '{}' was not deleted in the time allocated ({:.2}s).",pod_name,max_wait.as_secs_f64())]
+        WaitDeletionError { pod_name: String, max_wait: Duration },
+        #[error("")]
+        DeletePodsError { errors: Vec<String> },
+    }
+
+    #[derive(Debug, Error)]
+    pub enum EvictionError {
+        #[error("Evict Pod {} error: '{}'", pod_name, source)]
+        EvictionErrorRetry { source: kube::Error, pod_name: String },
+
+        #[error("Evict Pod {} error: '{}'", pod_name, source)]
+        EvictionErrorNoRetry { source: kube::Error, pod_name: String },
+    }
+}
diff --git a/KubeOS-Rust/proxy/src/main.rs b/KubeOS-Rust/proxy/src/main.rs
new file mode 100644
index 0000000000000000000000000000000000000000..c15aebed9e0c88eb7295e5e73f87347d337c6bc7
--- /dev/null
+++ b/KubeOS-Rust/proxy/src/main.rs
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+use anyhow::Result;
+use env_logger::{Builder, Env, Target};
+use futures::StreamExt;
+use kube::{
+    api::{Api, ListParams},
+    client::Client,
+    runtime::controller::{Context, Controller},
+};
+use log::{error, info};
+mod controller;
+use controller::{
+    error_policy, reconcile, AgentCallClient, AgentClient, ControllerClient, ProxyController, OS, SOCK_PATH,
+};
+
+const PROXY_VERSION: Option<&'static str> = option_env!("CARGO_PKG_VERSION");
+#[tokio::main]
+async fn main() -> Result<()> {
+    Builder::from_env(Env::default().default_filter_or("proxy=info")).target(Target::Stdout).init();
+    let client = Client::try_default().await?;
+    let os: Api<OS> = Api::all(client.clone());
+    let controller_client = ControllerClient::new(client.clone());
+    let agent_call_client = AgentCallClient::default();
+    let agent_client = AgentClient::new(SOCK_PATH, agent_call_client);
+    let proxy_controller = ProxyController::new(client, controller_client, agent_client);
+    info!("os-proxy version is {}, start renconcile", PROXY_VERSION.unwrap_or("Not Found"));
+    Controller::new(os, ListParams::default())
+        .run(reconcile, error_policy, Context::new(proxy_controller))
+        .for_each(|res| async move {
+            match res {
+                Ok(_o) => {},
+                Err(e) => error!("reconcile failed: {}", e.to_string()),
+            }
+        })
+        .await;
+    info!("os-proxy terminated");
+    Ok(())
+}
diff --git a/KubeOS-Rust/proxy/tests/common/mod.rs b/KubeOS-Rust/proxy/tests/common/mod.rs
new file mode 100644
index 0000000000000000000000000000000000000000..82577597ccd0a0fc7ff47ad035ab6b6e9a0c167d
--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/common/mod.rs
@@ -0,0 +1,63 @@
+use std::process::{Command, Stdio};
+
+use anyhow::Result;
+use k8s_openapi::api::core::v1::Node;
+use kube::{
+    api::ResourceExt,
+    client::Client,
+    config::{Config, KubeConfigOptions, Kubeconfig},
+    Api,
+};
+use manager::utils::{CommandExecutor, RealCommandExecutor};
+
+pub const CLUSTER: &str = "kubeos-test";
+
+pub fn run_command(cmd: &str, args: &[&str]) -> Result<()> {
+    let output = Command::new(cmd).args(args).stdout(Stdio::inherit()).stderr(Stdio::inherit()).output()?;
+    if !output.status.success() {
+        println!("failed to run command: {} {}\n", cmd, args.join(" "));
+    }
+    Ok(())
+}
+
+pub async fn setup() -> Result<Client> {
+    // set PATH variable
+    let path = std::env::var("PATH").unwrap();
+    let new_path = format!("{}:{}", path, "../../bin");
+    std::env::set_var("PATH", new_path);
+
+    // create cluster
+    let executor = RealCommandExecutor {};
+    println!("Creating cluster");
+    run_command("bash", &["./tests/setup/setup_test_env.sh"]).expect("failed to create cluster");
+
+    // connect to the cluster
+    let kind_config = executor.run_command_with_output("kind", &["get", "kubeconfig", "-n", CLUSTER]).unwrap();
+    let kubeconfig = Kubeconfig::from_yaml(kind_config.as_str()).expect("failed to parse kubeconfig");
+    let options = KubeConfigOptions::default();
+    let config = Config::from_custom_kubeconfig(kubeconfig, &&options).await.expect("failed to create config");
+    let client = Client::try_from(config).expect("failed to create client");
+    // list all nodes
+    let nodes: Api<Node> = Api::all(client.clone());
+    let node_list = nodes.list(&Default::default()).await.expect("failed to list nodes");
+    for n in node_list {
+        println!("Found Node: {}", n.name());
+    }
+    // check node status
+    let node = nodes.get("kubeos-test-worker").await.unwrap();
+    let status = node.status.unwrap();
+    let conditions = status.conditions.unwrap();
+    for c in conditions {
+        if c.type_ == "Ready" {
+            assert_eq!(c.status, "True");
+        }
+    }
+    println!("Cluster ready");
+    Ok(client)
+}
+
+pub fn clean_env() {
+    let executor = RealCommandExecutor {};
+    println!("Cleaning cluster");
+    executor.run_command("kind", &["delete", "clusters", CLUSTER]).expect("failed to clean cluster");
+}
diff --git a/KubeOS-Rust/proxy/tests/drain_test.rs b/KubeOS-Rust/proxy/tests/drain_test.rs
new file mode 100644
index 0000000000000000000000000000000000000000..2f4f1501310f76c4bca08335e868fc4ccac11e36
--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/drain_test.rs
@@ -0,0 +1,41 @@
+mod common;
+
+use common::*;
+use drain::drain_os;
+use k8s_openapi::api::core::v1::{Node, Pod};
+use kube::Api;
+
+#[tokio::test]
+#[ignore = "integration test"]
+async fn test_drain() {
+    let client = setup().await.unwrap();
+    // drain node
+    let nodes: Api<Node> = Api::all(client.clone());
+    let node_name = "kubeos-test-worker";
+    println!("cordon node");
+    nodes.cordon(node_name).await.unwrap();
+    println!("drain node");
+    drain_os(&client, node_name, true).await.unwrap();
+
+    // assert unschedulable
+    println!("check node unschedulable");
+    let node = nodes.get(node_name).await.unwrap();
+    if let Some(spec) = node.spec {
+        assert_eq!(spec.unschedulable, Some(true));
+    } else {
+        panic!("node spec is none");
+    }
+    // list all pods on kubeos-test-worker node and all pods should belong to daemonset
+    println!("list all pods on kubeos-test-worker node");
+    let pods: Api<Pod> = Api::all(client.clone());
+    let pod_list = pods.list(&Default::default()).await.unwrap();
+    // check the pod is from daemonset
+    for p in pod_list {
+        if p.spec.unwrap().node_name.unwrap() == node_name {
+            assert_eq!(p.metadata.owner_references.unwrap()[0].kind, "DaemonSet");
+        }
+    }
+    nodes.uncordon(node_name).await.unwrap();
+
+    clean_env()
+}
diff --git a/KubeOS-Rust/proxy/tests/setup/kind-config.yaml b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0fe29e735b82d8998f68719a42782c3d74d76163
--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/setup/kind-config.yaml
@@ -0,0 +1,5 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+- role: worker
\ No newline at end of file
diff --git a/KubeOS-Rust/proxy/tests/setup/resources.yaml b/KubeOS-Rust/proxy/tests/setup/resources.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0e449d5d3664a080512d729f8cff898aed59995f
--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/setup/resources.yaml
@@ -0,0 +1,102 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: example-daemonset
+spec:
+  selector:
+    matchLabels:
+      name: example-daemonset
+  template:
+    metadata:
+      labels:
+        name: example-daemonset
+    spec:
+      containers:
+      - name: busybox
+        image: busybox:stable
+        command: ["/bin/sh", "-c", "sleep 3600"]
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: pod-with-local-storage
+spec:
+  containers:
+  - name: busybox
+    image: busybox:stable
+    command: ["/bin/sh", "-c", "sleep 3600"]
+    volumeMounts:
+    - mountPath: "/data"
+      name: local-volume
+  volumes:
+  - name: local-volume
+    emptyDir: {}
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: standalone-pod
+spec:
+  containers:
+  - name: busybox
+    image: busybox:stable
+    command: ["/bin/sh", "-c", "sleep 3600"]
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: example-deployment
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: example
+  template:
+    metadata:
+      labels:
+        app: example
+    spec:
+      containers:
+      - name: nginx
+        image: nginx:alpine
+      affinity:
+        nodeAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 1
+            preference:
+              matchExpressions:
+              - key: "node-role.kubernetes.io/control-plane"
+                operator: DoesNotExist
+      tolerations:
+      - key: "node-role.kubernetes.io/master"
+        operator: "Exists"
+      - key: "node-role.kubernetes.io/control-plane"
+        operator: "Exists"
+---
+apiVersion: policy/v1
+kind: PodDisruptionBudget
+metadata:
+  name: example-pdb
+spec:
+  minAvailable: 1
+  selector:
+    matchLabels:
+      app: example
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: resource-intensive-pod
+spec:
+  containers:
+  - name: busybox
+    image: busybox:stable
+    command: ["/bin/sh", "-c", "sleep 3600"]
+    resources:
+      requests:
+        memory: "256Mi"
+        cpu: "500m"
+      limits:
+        memory: "512Mi"
+        cpu: "1000m"
+
diff --git a/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh
new file mode 100644
index 0000000000000000000000000000000000000000..d24d8e01a208fb1ab2fdc17b5fb5ff2b933a6b4b
--- /dev/null
+++ b/KubeOS-Rust/proxy/tests/setup/setup_test_env.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+# this bash script executes in proxy directory
+
+set -Eeuxo pipefail
+
+# Define variables
+KIND_VERSION="v0.19.0"
+KUBECTL_VERSION="v1.24.15"
+KIND_CLUSTER_NAME="kubeos-test"
+DOCKER_IMAGES=("busybox:stable" "nginx:alpine" "kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab")
+NODE_IMAGE="kindest/node:v1.24.15@sha256:7db4f8bea3e14b82d12e044e25e34bd53754b7f2b0e9d56df21774e6f66a70ab"
+RESOURCE="./tests/setup/resources.yaml"
+KIND_CONFIG="./tests/setup/kind-config.yaml"
+BIN_PATH="../../bin/"
+ARCH=$(uname -m)
+
+# Install kind and kubectl
+install_bins() {
+    # if bin dir not exist then create
+    if [ ! -d "${BIN_PATH}" ]; then
+        mkdir -p "${BIN_PATH}"
+    fi
+    if [ ! -f "${BIN_PATH}"kind ]; then
+        echo "Installing Kind..."
+        # For AMD64 / x86_64
+        if [ "$ARCH" = x86_64 ]; then
+            # add proxy if you are behind proxy
+            curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-amd64
+        fi
+        # For ARM64
+        if [ "$ARCH" = aarch64 ]; then
+            curl -Lo "${BIN_PATH}"kind https://kind.sigs.k8s.io/dl/"${KIND_VERSION}"/kind-linux-arm64
+        fi
+        chmod +x "${BIN_PATH}"kind
+    fi
+    if [ ! -f "${BIN_PATH}"kubectl ]; then
+        echo "Installing kubectl..."
+        if [ "$ARCH" = x86_64 ]; then
+            curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
+        fi
+        if [ "$ARCH" = aarch64 ]; then
+            curl -Lo "${BIN_PATH}"kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/arm64/kubectl"
+        fi
+        chmod +x "${BIN_PATH}"kubectl
+    fi
+    export PATH=$PATH:"${BIN_PATH}"
+}
+
+# Create Kind Cluster
+create_cluster() {
+    echo "Creating Kind cluster..."
+    for image in "${DOCKER_IMAGES[@]}"; do
+        docker pull "$image"
+    done
+    kind create cluster --name "${KIND_CLUSTER_NAME}" --config "${KIND_CONFIG}" --image "${NODE_IMAGE}"
+}
+
+# Load Docker image into Kind cluster
+load_docker_image() {
+    echo "Loading Docker image into Kind cluster..."
+    DOCKER_IMAGE=$(printf "%s " "${DOCKER_IMAGES[@]:0:2}")
+    kind load docker-image ${DOCKER_IMAGE} --name "${KIND_CLUSTER_NAME}"
+}
+
+# Apply Kubernetes resource files
+apply_k8s_resources() {
+    echo "Applying Kubernetes resources from ${RESOURCE}..."
+    kubectl apply -f "${RESOURCE}"
+    echo "Waiting for nodes getting ready..."
+    sleep 40s
+}
+
+main() {
+    export no_proxy=localhost,127.0.0.1
+    install_bins
+    create_cluster
+    load_docker_image
+    apply_k8s_resources
+}
+
+main
diff --git a/KubeOS-Rust/rustfmt.toml b/KubeOS-Rust/rustfmt.toml
new file mode 100644
index 0000000000000000000000000000000000000000..3c565cf53340471a5ca004302bec88efe3bcc3cd
--- /dev/null
+++ b/KubeOS-Rust/rustfmt.toml
@@ -0,0 +1,11 @@
+# cargo +nightly fmt
+version = "Two"
+use_small_heuristics = "MAX"
+match_block_trailing_comma = true
+newline_style = "Unix"
+merge_derives = false
+max_width = 120
+group_imports = "StdExternalCrate"
+imports_granularity = "Crate"
+reorder_imports = true
+unstable_features = true
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 634a3bcf83108786d8ee669ca45f8e7ac4675e3b..34e27aaa3817f41ac545f8fe8a1805467edbd56d 100644
--- a/Makefile
+++ b/Makefile
@@ -46,11 +46,13 @@ GO_BUILD_CGO = CGO_ENABLED=1 \
 	CGO_LDFLAGS="-Wl,-z,relro,-z,now -Wl,-z,noexecstack" \
 	${GO_BUILD} -buildmode=pie -trimpath -tags "seccomp selinux static_build cgo netgo osusergo"
 
-all: proxy operator agent hostshell
+RUSTFLAGS := RUSTFLAGS="-C relocation_model=pic -D warnings -W unsafe_code -W rust_2021_incompatible_closure_captures -C link-arg=-s"
+
+all: proxy operator agent hostshell rust-kubeos
 
 # Build binary
 proxy:
-	${GO_BUILD_CGO} ${LD_FLAGS} -o bin/proxy  cmd/proxy/main.go
+	${GO_BUILD_CGO} ${LD_FLAGS} -o bin/proxy cmd/proxy/main.go
 	strip bin/proxy
 
 operator:
@@ -69,9 +71,28 @@ hostshell:
 	${GO_BUILD_CGO} ${LD_FLAGS} -o bin/hostshell cmd/admin-container/main.go
 	strip bin/hostshell
 
+rust-kubeos:
+	cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust
+
+rust-proxy:
+	cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package proxy
+
+rust-agent:
+	cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package os-agent
+
+rust-operator:
+	cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package operator
+
+rust-kbimg:
+	cd KubeOS-Rust && ${RUSTFLAGS} cargo build --profile release --target-dir ../bin/rust --package kbimg
+
+# clean binary
+clean:
+	rm -rf bin
+
 # Install CRDs into a cluster
 install: manifests
-	kubectl apply -f confg/crd
+	kubectl apply -f config/crd
 
 # Uninstall CRDs from a cluster
 uninstall: manifests
@@ -104,7 +125,7 @@ generate: controller-gen
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
 # Build the docker image
-docker-build: operator proxy
+docker-build: operator rust-proxy
 	docker build --target operator -t ${IMG_OPERATOR} .
 	docker build --target proxy -t ${IMG_PROXY} .
 
diff --git a/VERSION b/VERSION
index ee90284c27f187a315f1267b063fa81b5b84f613..af0b7ddbffd5d5a877581ba8ba10deed158da14d 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.0.4
+1.0.6
diff --git a/api/v1alpha1/os_types.go b/api/v1alpha1/os_types.go
index d3d636de893826318d96d52a0a77901f68006ae3..bde86d251a051cbc9654ca173458f75d82557986 100644
--- a/api/v1alpha1/os_types.go
+++ b/api/v1alpha1/os_types.go
@@ -24,10 +24,12 @@ type OSSpec struct {
 	CheckSum       string `json:"checksum"`
 	FlagSafe       bool   `json:"flagSafe"`
 	MTLS           bool   `json:"mtls"`
+	// +kubebuilder:validation:Enum=docker;disk;containerd
 	ImageType      string `json:"imagetype"`
 	ContainerImage string `json:"containerimage"`
-	OpsType        string `json:"opstype"`
-	EvictPodForce  bool   `json:"evictpodforce"`
+	// +kubebuilder:validation:Enum=upgrade;config;rollback
+	OpsType       string `json:"opstype"`
+	EvictPodForce bool   `json:"evictpodforce"`
 	// +kubebuilder:validation:Optional
 	CaCert string `json:"cacert"`
 	// +kubebuilder:validation:Optional
@@ -39,7 +41,17 @@ type OSSpec struct {
 	// +kubebuilder:validation:Optional
 	UpgradeConfigs SysConfigs `json:"upgradeconfigs"`
 	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:=no-label
 	NodeSelector string `json:"nodeselector"`
+	// +kubebuilder:validation:Optional
+	TimeWindow TimeWindow `json:"timewindow"`
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:=15
+	TimeInterval int `json:"timeinterval"`
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:validation:Enum=serial;parallel
+	// +kubebuilder:default:=parallel
+	ExecutionMode string `json:"executionmode"`
 }
 
 // +kubebuilder:subresource:status
@@ -90,6 +102,11 @@ type Content struct {
 	Operation string `json:"operation"`
 }
 
+type TimeWindow struct {
+	StartTime string `json:"starttime"`
+	EndTime   string `json:"endtime"`
+}
+
 // +kubebuilder:subresource:status
 // +kubebuilder:object:root=true
 
@@ -116,11 +133,21 @@ type OSInstanceSpec struct {
 	// +kubebuilder:validation:Optional
 	NodeStatus string `json:"nodestatus"`
 	// +kubebuilder:validation:Optional
+	NamespacedName NamespacedName `json:"namespacedname"`
+	// +kubebuilder:validation:Optional
 	SysConfigs SysConfigs `json:"sysconfigs"`
 	// +kubebuilder:validation:Optional
 	UpgradeConfigs SysConfigs `json:"upgradeconfigs"`
 }
 
+// NamespacedName defines name and namespace of os corresponding to osinstance
+type NamespacedName struct {
+	// +kubebuilder:validation:Optional
+	Name string `json:"name"`
+	// +kubebuilder:validation:Optional
+	Namespace string `json:"namespace"`
+}
+
 // +kubebuilder:object:root=true
 
 // OSInstanceList is a list of OSInstance
diff --git a/cmd/operator/controllers/operation.go b/cmd/operator/controllers/operation.go
index 9c9c6426b688090457d0fca87974bca0528eead3..9f1304796193399859cce24c9b603ffdb216ad10 100644
--- a/cmd/operator/controllers/operation.go
+++ b/cmd/operator/controllers/operation.go
@@ -15,50 +15,37 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apimachinery/pkg/types"
-	"openeuler.org/KubeOS/pkg/common"
-	"openeuler.org/KubeOS/pkg/values"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	upgradev1 "openeuler.org/KubeOS/api/v1alpha1"
+	"openeuler.org/KubeOS/pkg/common"
+	"openeuler.org/KubeOS/pkg/values"
 )
 
 type operation interface {
-	newExistRequirement() (labels.Requirement, error)
-	newNotExistRequirement() (labels.Requirement, error)
 	updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS,
-		nodes []corev1.Node, limit int) (int, error)
+		nodes []corev1.Node, limit int) (int, []error)
 	updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS,
 		node *corev1.Node, osInstance *upgradev1.OSInstance) error
+	getOpsLabel() opsLabel
 }
 
-type upgradeOps struct{}
-
-func (u upgradeOps) newExistRequirement() (labels.Requirement, error) {
-	requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.Exists, nil)
-	if err != nil {
-		log.Error(err, "unable to create requirement "+values.LabelUpgrading)
-		return labels.Requirement{}, err
-	}
-	return *requirement, nil
+type upgradeOps struct {
+	label opsLabel
 }
 
-func (u upgradeOps) newNotExistRequirement() (labels.Requirement, error) {
-	requirement, err := labels.NewRequirement(values.LabelUpgrading, selection.DoesNotExist, nil)
-	if err != nil {
-		log.Error(err, "unable to create requirement "+values.LabelUpgrading)
-		return labels.Requirement{}, err
-	}
-	return *requirement, nil
+func (u upgradeOps) getOpsLabel() opsLabel {
+	return u.label
 }
 
 func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS,
-	nodes []corev1.Node, limit int) (int, error) {
-	var count int
+	nodes []corev1.Node, limit int) (int, []error) {
+	var count = 0
+	var errList []error
 	for _, node := range nodes {
 		if count >= limit {
 			break
@@ -67,21 +54,52 @@ func (u upgradeOps) updateNodes(ctx context.Context, r common.ReadStatusWriter,
 		if os.Spec.OSVersion != osVersionNode {
 			log.Info("Upgrading node " + node.Name)
 			var osInstance upgradev1.OSInstance
-			if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil {
+			if err := r.Get(ctx, types.NamespacedName{Namespace: values.OsiNamespace, Name: node.Name}, &osInstance); err != nil {
 				if err = client.IgnoreNotFound(err); err != nil {
-					log.Error(err, "failed to get osInstance "+node.Name, "skip this node")
-					return count, err
+					log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+
+						"Restart the reconcile and wait until it is complete.")
+					return count, []error{err}
 				}
+				log.Error(err, "failed to get osInstance "+node.Name+"skip this node")
+				errList = append(errList, err)
 				continue
 			}
 			if err := u.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil {
 				log.Error(err, "failed to update node and osinstance ,skip this node ")
+				errList = append(errList, err)
 				continue
 			}
 			count++
 		}
 	}
+	if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial {
+		if errList = deleteSerialLabel(ctx, r, nodes); len(errList) != 0 {
+			log.Error(nil, "failed to delete nodes serial label")
+		}
+	}
+	if len(errList) > 0 {
+		return count, errList
+	}
 	return count, nil
+
+}
+
+func deleteSerialLabel(ctx context.Context, r common.ReadStatusWriter, nodes []corev1.Node) []error {
+	var errList []error
+	for _, node := range nodes {
+		if _, ok := node.Labels[values.LabelSerial]; ok {
+			delete(node.Labels, values.LabelSerial)
+			if err := r.Update(ctx, &node); err != nil {
+				log.Error(err, "unable to delete serial label ", "node", node.Name+", skip this node")
+				errList = append(errList, err)
+			}
+			log.Info("delete node " + node.Name + " serial label " + values.LabelSerial + " successfully")
+		}
+	}
+	if len(errList) > 0 {
+		return errList
+	}
+	return nil
 }
 
 func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS,
@@ -106,11 +124,14 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW
 		}
 	}
 	osInstance.Spec.NodeStatus = values.NodeStatusUpgrade.String()
+	osInstance.Spec.NamespacedName = upgradev1.NamespacedName{Name: os.Name, Namespace: os.Namespace}
+	log.V(1).Info("Wait to update osinstance name:" + osInstance.Name + " node name is " + node.Name)
 	if err := r.Update(ctx, osInstance); err != nil {
 		log.Error(err, "unable to update", "osInstance", osInstance.Name)
 		return err
 	}
 	log.Info("Update osinstance spec successfully")
+
 	node.Labels[values.LabelUpgrading] = ""
 	if err := r.Update(ctx, node); err != nil {
 		log.Error(err, "unable to label", "node", node.Name)
@@ -120,29 +141,18 @@ func (u upgradeOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusW
 	return nil
 }
 
-type configOps struct{}
-
-func (c configOps) newExistRequirement() (labels.Requirement, error) {
-	requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.Exists, nil)
-	if err != nil {
-		log.Error(err, "unable to create requirement "+values.LabelConfiguring)
-		return labels.Requirement{}, err
-	}
-	return *requirement, nil
+type configOps struct {
+	label opsLabel
 }
 
-func (c configOps) newNotExistRequirement() (labels.Requirement, error) {
-	requirement, err := labels.NewRequirement(values.LabelConfiguring, selection.DoesNotExist, nil)
-	if err != nil {
-		log.Error(err, "unable to create requirement "+values.LabelConfiguring)
-		return labels.Requirement{}, err
-	}
-	return *requirement, nil
+func (c configOps) getOpsLabel() opsLabel {
+	return c.label
 }
 
 func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS,
-	nodes []corev1.Node, limit int) (int, error) {
-	var count int
+	nodes []corev1.Node, limit int) (int, []error) {
+	var count = 0
+	var errList []error
 	for _, node := range nodes {
 		if count >= limit {
 			break
@@ -150,21 +160,34 @@ func (c configOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, o
 		var osInstance upgradev1.OSInstance
 		if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil {
 			if err = client.IgnoreNotFound(err); err != nil {
-				log.Error(err, "failed to get osInstance "+node.Name)
-				return count, err
+				log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+
+					"Restart the reconcile and wait until it is complete.")
+				return count, []error{err}
 			}
+			log.Error(err, "failed to get osInstance "+node.Name+", skip this node")
+			errList = append(errList, err)
 			continue
 		}
 		if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version {
 			log.Info("Configuring node " + node.Name)
 			if err := c.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil {
 				log.Error(err, "failed to update node and osinstance ,skip this node ")
+				errList = append(errList, err)
 				continue
 			}
 			count++
 		}
 	}
-	return count, nil
+	if count == 0 && os.Spec.ExecutionMode == ExecutionModeSerial {
+		if errList = deleteSerialLabel(ctx, r, nodes); len(errList) != 0 {
+			log.Error(nil, "failed to delete nodes serial label")
+		}
+	}
+	if len(errList) > 0 {
+		return count, errList
+	}
+	return count, errList
+
 }
 
 func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS,
@@ -173,11 +196,14 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr
 		return err
 	}
 	osInstance.Spec.NodeStatus = values.NodeStatusConfig.String()
+	osInstance.Spec.NamespacedName = upgradev1.NamespacedName{Name: os.Name, Namespace: os.Namespace}
+	log.V(1).Info("Wait to update osinstance name:" + osInstance.Name + " node name is " + node.Name)
 	if err := r.Update(ctx, osInstance); err != nil {
 		log.Error(err, "unable to update", "osInstance", osInstance.Name)
 		return err
 	}
 	log.Info("Update osinstance spec successfully")
+
 	node.Labels[values.LabelConfiguring] = ""
 	if err := r.Update(ctx, node); err != nil {
 		log.Error(err, "unable to label", "node", node.Name)
@@ -186,3 +212,72 @@ func (c configOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWr
 	log.Info("Add node configuring label " + values.LabelConfiguring + " successfully")
 	return nil
 }
+
+type serialOps struct {
+	label opsLabel
+}
+
+func (s serialOps) getOpsLabel() opsLabel {
+	return s.label
+}
+
+func (s serialOps) updateNodes(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS,
+	nodes []corev1.Node, limit int) (int, []error) {
+	var count int
+	var errList []error
+	for _, node := range nodes {
+		if count >= limit {
+			break
+		}
+		var osInstance upgradev1.OSInstance
+		if err := r.Get(ctx, types.NamespacedName{Namespace: os.GetObjectMeta().GetNamespace(), Name: node.Name}, &osInstance); err != nil {
+			if err = client.IgnoreNotFound(err); err != nil {
+				log.Error(err, "osInstance not found "+node.Name, ", maybe the os-proxy initialization is not complete. "+
+					"Restart the reconcile and wait until it is complete.")
+				return count, []error{err}
+			}
+			log.Error(err, "failed to get osInstance "+node.Name+", skip this node")
+			errList = append(errList, err)
+			continue
+		}
+		switch s.getOpsLabel().label {
+		case values.LabelUpgrading:
+			if os.Spec.OSVersion != node.Status.NodeInfo.OSImage {
+				log.Info("Add Serial Label to node " + node.Name)
+				if err := s.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil {
+					log.Error(err, "failed to update node and osinstance ,skip this node ")
+					errList = append(errList, err)
+					continue
+				}
+				count++
+			}
+		case values.LabelConfiguring:
+			if os.Spec.SysConfigs.Version != osInstance.Spec.SysConfigs.Version {
+				log.Info("Add Serial Label to node " + node.Name)
+				if err := s.updateNodeAndOSins(ctx, r, os, &node, &osInstance); err != nil {
+					log.Error(err, "failed to update node and osinstance ,skip this node ")
+					errList = append(errList, err)
+					continue
+				}
+				count++
+			}
+		default:
+			log.Error(nil, "ops "+s.getOpsLabel().label+" cannot be recognized")
+			return count, []error{fmt.Errorf("ops " + s.getOpsLabel().label + " cannot be recognized")}
+		}
+	}
+	if len(errList) == 0 {
+		return count, nil
+	}
+	return count, errList
+}
+func (s serialOps) updateNodeAndOSins(ctx context.Context, r common.ReadStatusWriter, os *upgradev1.OS,
+	node *corev1.Node, osInstance *upgradev1.OSInstance) error {
+	node.Labels[values.LabelSerial] = ""
+	if err := r.Update(ctx, node); err != nil {
+		log.Error(err, "unable to label", "node", node.Name)
+		return err
+	}
+	log.Info("Add node serial label " + values.LabelSerial + " successfully")
+	return nil
+}
diff --git a/cmd/operator/controllers/os_controller.go b/cmd/operator/controllers/os_controller.go
index 2c4ee1f7b3a80df497a58a2ca5d3df811ecda902..adcae87da60913ee2bd72bcdd1f79611bdfce2f4 100644
--- a/cmd/operator/controllers/os_controller.go
+++ b/cmd/operator/controllers/os_controller.go
@@ -17,6 +17,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"strconv"
+	"time"
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -56,35 +58,71 @@ func (r *OSReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Re
 
 // Reconcile compares the actual state with the desired and updates the status of the resources e.g. nodes
 func Reconcile(ctx context.Context, r common.ReadStatusWriter, req ctrl.Request) (ctrl.Result, error) {
-	os, nodeNum, err := getAndUpdateOS(ctx, r, req.NamespacedName)
+	log.V(1).Info("start Reconcile of " + req.Name + " with namespace is " + req.Namespace)
+	os, err := getOSCr(ctx, r, req.NamespacedName)
 	if err != nil {
 		if errors.IsNotFound(err) {
 			return values.NoRequeue, nil
 		}
 		return values.RequeueNow, err
 	}
-
+	isWithinTimeWindow, err := isWithinTimeWindow(os.Spec.TimeWindow.StartTime, os.Spec.TimeWindow.EndTime)
+	if err != nil {
+		return values.RequeueNow, err
+	}
+	if !isWithinTimeWindow {
+		log.V(1).Info("not in time window, the start time is " + os.Spec.TimeWindow.StartTime +
+			" , the end time " + os.Spec.TimeWindow.EndTime)
+		return values.Requeue, nil
+	}
 	ops := os.Spec.OpsType
 	var opsInsatnce operation
 	switch ops {
 	case "upgrade", "rollback":
-		opsInsatnce = upgradeOps{}
+		opsInsatnce = upgradeOps{
+			label: opsLabel{
+				label: values.LabelUpgrading,
+				op:    selection.DoesNotExist,
+			},
+		}
 	case "config":
-		opsInsatnce = configOps{}
+		opsInsatnce = configOps{
+			label: opsLabel{
+				label: values.LabelConfiguring,
+				op:    selection.DoesNotExist,
+			},
+		}
 	default:
 		log.Error(nil, "operation "+ops+" cannot be recognized")
 		return values.Requeue, nil
 	}
-	limit, err := calNodeLimit(ctx, r, opsInsatnce, min(os.Spec.MaxUnavailable, nodeNum), os.Spec.NodeSelector) // adjust maxUnavailable if need
+	commonNodesReq, err := newCommonsNodesRequirement(os.Spec.NodeSelector,
+		selection.Equals).createNodeRequirement(ctx, r)
 	if err != nil {
 		return values.RequeueNow, err
 	}
-	if needRequeue, err := assignOperation(ctx, r, os, limit, opsInsatnce); err != nil {
+	allNodes, err := getNodes(ctx, r, 0, commonNodesReq...)
+	if err != nil {
 		return values.RequeueNow, err
-	} else if needRequeue {
+	}
+	log.V(1).Info("get all nodes num is " + strconv.Itoa(len(allNodes)))
+	switch os.Spec.ExecutionMode {
+	case ExecutionModeParallel:
+		result, err := excuteParallelOperation(ctx, r, os, opsInsatnce, len(allNodes))
+		if err != nil {
+			return values.RequeueNow, nil
+		}
+		return result, nil
+	case ExecutionModeSerial:
+		result, err := excuteSerialOperation(ctx, r, os, opsInsatnce, len(allNodes))
+		if err != nil {
+			return values.RequeueNow, err
+		}
+		return result, nil
+	default:
+		log.Error(nil, "excutionMode "+os.Spec.ExecutionMode+" cannot be recognized")
 		return values.Requeue, nil
 	}
-	return values.Requeue, nil
 }
 
 // SetupWithManager sets up the controller with the Manager.
@@ -124,72 +162,17 @@ func (r *OSReconciler) DeleteOSInstance(e event.DeleteEvent, q workqueue.RateLim
 	}
 }
 
-func getAndUpdateOS(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS,
-	int, error) {
+func getOSCr(ctx context.Context, r common.ReadStatusWriter, name types.NamespacedName) (upgradev1.OS, error) {
 	var os upgradev1.OS
 	if err := r.Get(ctx, name, &os); err != nil {
 		log.Error(err, "unable to fetch OS")
-		return upgradev1.OS{}, 0, err
-	}
-
-	requirement, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil)
-	if err != nil {
-		log.Error(err, "unable to create requirement "+values.LabelMaster)
-		return upgradev1.OS{}, 0, err
-	}
-	var requirements []labels.Requirement
-	requirements = append(requirements, *requirement)
-	if os.Spec.NodeSelector != "" {
-		reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Exists, nil)
-		if err != nil {
-			log.Error(err, "unable to create requirement "+values.LabelNodeSelector)
-			return upgradev1.OS{}, 0, err
-		}
-		requirements = append(requirements, *requirement, *reqSelector)
-	}
-	nodesItems, err := getNodes(ctx, r, 0, requirements...)
-	if err != nil {
-		log.Error(err, "get slave nodes fail")
-		return upgradev1.OS{}, 0, err
-	}
-	nodeNum := len(nodesItems)
-	return os, nodeNum, nil
-}
-
-func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int,
-	ops operation) (bool, error) {
-	requirement, err := ops.newNotExistRequirement()
-	if err != nil {
-		log.Error(err, "unable to create requirement "+values.LabelUpgrading)
-		return false, err
-	}
-	reqMaster, err := labels.NewRequirement(values.LabelMaster, selection.DoesNotExist, nil)
-	if err != nil {
-		log.Error(err, "unable to create requirement "+values.LabelMaster)
-		return false, err
-	}
-	var requirements []labels.Requirement
-	requirements = append(requirements, requirement, *reqMaster)
-	if os.Spec.NodeSelector != "" {
-		reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{os.Spec.NodeSelector})
-		if err != nil {
-			log.Error(err, "unable to create requirement "+values.LabelNodeSelector)
-			return false, err
-		}
-		requirements = append(requirements, *reqSelector)
-	}
-
-	nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated
-	if err != nil {
-		return false, err
+		return upgradev1.OS{}, err
 	}
-	// Upgrade OS for selected nodes
-	count, err := ops.updateNodes(ctx, r, &os, nodes, limit)
-	if err != nil {
-		return false, err
+	if err := checkNodeSelector(ctx, r, os); err != nil {
+		log.Error(err, "nodeselector conficts")
+		return upgradev1.OS{}, err
 	}
-
-	return count >= limit, nil
+	return os, nil
 }
 
 func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int,
@@ -204,28 +187,30 @@ func getNodes(ctx context.Context, r common.ReadStatusWriter, limit int,
 }
 
 func calNodeLimit(ctx context.Context, r common.ReadStatusWriter,
-	ops operation, maxUnavailable int, nodeSelector string) (int, error) {
-	requirement, err := ops.newExistRequirement()
+	maxUnavailable int, requirements []labels.Requirement) (int, error) {
+	nodes, err := getNodes(ctx, r, 0, requirements...)
 	if err != nil {
-		log.Error(err, "unable to create requirement "+values.LabelUpgrading)
 		return 0, err
+
 	}
-	var requirements []labels.Requirement
-	requirements = append(requirements, requirement)
-	if nodeSelector != "" {
-		reqSelector, err := labels.NewRequirement(values.LabelNodeSelector, selection.Equals, []string{nodeSelector})
-		if err != nil {
-			log.Error(err, "unable to create requirement "+values.LabelNodeSelector)
-			return 0, err
-		}
-		requirements = append(requirements, *reqSelector)
+	return maxUnavailable - len(nodes), nil
+}
+func assignOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS, limit int,
+	opsInstance operation, requirements []labels.Requirement) (int, error) {
+	if limit == 0 {
+		log.V(1).Info("limit is 0 , do not need to assign operation")
+		return 0, nil
 	}
-	nodes, err := getNodes(ctx, r, 0, requirements...)
+	nodes, err := getNodes(ctx, r, limit+1, requirements...) // one more to see if all nodes updated
 	if err != nil {
 		return 0, err
-
 	}
-	return maxUnavailable - len(nodes), nil
+	log.V(1).Info("get wait to check nodes is " + strconv.Itoa(len(nodes)))
+	count, errLists := opsInstance.updateNodes(ctx, r, &os, nodes, limit)
+	if len(errLists) != 0 {
+		return 0, fmt.Errorf("update nodes and osinstance error")
+	}
+	return count, nil
 }
 
 func min(a, b int) int {
@@ -259,3 +244,123 @@ func deepCopySpecConfigs(os *upgradev1.OS, osinstance *upgradev1.OSInstance, con
 	}
 	return nil
 }
+
+// Check whether the nodeselector conflicts with other nodeselector in OS CRs. If the nodeselector is empty, return the list of other nodeselectors.
+func checkNodeSelector(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS) error {
+	var osList upgradev1.OSList
+	if err := r.List(ctx, &osList, &client.ListOptions{}); err != nil {
+		log.Error(err, "unable to list nodes with requirements")
+		return err
+	}
+	var sameNodeSelectorList []types.NamespacedName
+	for _, osItem := range osList.Items {
+		// Exclude current os, controller-runtime not supports multiple indexs as listoptions in current version,
+		// so cannot list os without current os use List function
+		if osItem.Name == os.Name && osItem.Namespace == os.Namespace {
+			continue
+		}
+		if os.Spec.NodeSelector == osItem.Spec.NodeSelector {
+			sameNodeSelectorList = append(sameNodeSelectorList, types.NamespacedName{
+				Namespace: osItem.Namespace,
+				Name:      osItem.Name,
+			})
+		}
+	}
+	// If a node label corresponds to multiple OS CRs, upgrade or configuration information may conflict.
+	// As a result, an error is reported and returned when there are one-to-many relationships.
+	if len(sameNodeSelectorList) > 0 {
+		errorMessage := sameNodeSelectorList[0].String()
+		for i := 1; i < len(sameNodeSelectorList); i++ {
+			errorMessage = errorMessage + " , " + sameNodeSelectorList[i].String()
+		}
+		log.Error(nil, "OS CR "+os.Name+" in namespace "+os.Namespace+" has same nodeselector with "+errorMessage)
+		return fmt.Errorf("OS CR %s in namespace %s has same nodeselector with %s", os.Name, os.Namespace, errorMessage)
+	}
+	return nil
+}
+
+func setTimeInterval(timeInterval int) ctrl.Result {
+	return ctrl.Result{Requeue: true, RequeueAfter: time.Duration(timeInterval) * time.Second}
+}
+
+func excuteParallelOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS,
+	opsInsatnce operation, nodeNum int) (ctrl.Result, error) {
+	log.V(1).Info("start parallel operation")
+	opsLabel := opsInsatnce.getOpsLabel()
+	opsLabel.op = selection.Exists
+	opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector,
+		selection.Equals, opsLabel).createNodeRequirement(ctx, r)
+	if err != nil {
+		return values.RequeueNow, nil
+	}
+	limit, err := calNodeLimit(ctx, r, min(os.Spec.MaxUnavailable, nodeNum), opsNodesReq) // adjust maxUnavailable if need
+	if err != nil {
+		return values.RequeueNow, nil
+	}
+	log.V(1).Info("get limit is " + strconv.Itoa(limit))
+	opsLabel.op = selection.DoesNotExist
+	noOpsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector,
+		selection.Equals, opsLabel).createNodeRequirement(ctx, r)
+	if err != nil {
+		return values.RequeueNow, nil
+	}
+	if _, err := assignOperation(ctx, r, os, limit, opsInsatnce, noOpsNodesReq); err != nil {
+		return values.RequeueNow, nil
+	}
+	return setTimeInterval(os.Spec.TimeInterval), nil
+}
+
+func excuteSerialOperation(ctx context.Context, r common.ReadStatusWriter, os upgradev1.OS,
+	opsInsatnce operation, nodeNum int) (ctrl.Result, error) {
+	log.V(1).Info("start serial operation")
+	opsLabel := opsInsatnce.getOpsLabel()
+	opsLabel.op = selection.Exists
+	opsNodesReq, err := newopsNodesRequirement(os.Spec.NodeSelector,
+		selection.Equals, opsLabel).createNodeRequirement(ctx, r)
+	if err != nil {
+		return values.RequeueNow, nil
+	}
+	opsNodeNum, err := getNodes(ctx, r, 0, opsNodesReq...)
+	if err != nil {
+		return values.RequeueNow, nil
+	}
+	if len(opsNodeNum) > 0 {
+		log.V(1).Info("a node is being upgraded or configured. Wait until the node upgrade or configuration is complete.")
+		return values.Requeue, nil
+	}
+
+	serialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector,
+		selection.Equals, selection.Exists).createNodeRequirement(ctx, r)
+	if err != nil {
+		return values.RequeueNow, nil
+	}
+	serialNodeLimit, err := calNodeLimit(ctx, r, min(os.Spec.MaxUnavailable, nodeNum), serialNodesRequirement)
+	if err != nil {
+		return values.RequeueNow, nil
+	}
+	log.V(1).Info("get the number of nodes which need to be added serial label num is " + strconv.Itoa(serialNodeLimit))
+	noSerialNodesRequirement, err := newSerialNodesRequirement(os.Spec.NodeSelector,
+		selection.Equals, selection.DoesNotExist).createNodeRequirement(ctx, r)
+	if err != nil {
+		return values.RequeueNow, nil
+	}
+	// add serial label to node
+	serialOpsInstance := serialOps{
+		label: opsInsatnce.getOpsLabel(),
+	}
+	log.V(1).Info("start add serial label to nodes")
+	if _, err := assignOperation(ctx, r, os, serialNodeLimit, serialOpsInstance, noSerialNodesRequirement); err != nil {
+		return values.RequeueNow, nil
+	}
+
+	log.V(1).Info("start check nodes needed to be upgrade/configure or not")
+	serialLimit := 1 // 1 is the number of operation nodes when excution mode in serial
+	count, err := assignOperation(ctx, r, os, serialLimit, opsInsatnce, serialNodesRequirement)
+	if err != nil {
+		return values.RequeueNow, nil
+	}
+	if count > 0 {
+		return values.Requeue, nil
+	}
+	return setTimeInterval(os.Spec.TimeInterval), nil
+}
diff --git a/cmd/operator/controllers/requirements.go b/cmd/operator/controllers/requirements.go
new file mode 100644
index 0000000000000000000000000000000000000000..12db8e8242f28c6f54425a5306cd2993f0a2c3e7
--- /dev/null
+++ b/cmd/operator/controllers/requirements.go
@@ -0,0 +1,199 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+// Package controllers contains the Reconcile of operator
+package controllers
+
+import (
+	"context"
+
+	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/selection"
+
+	"openeuler.org/KubeOS/pkg/common"
+	"openeuler.org/KubeOS/pkg/values"
+)
+
+const (
+	AllNodeSelector = "all-label"
+	NoNodeSelector  = "no-label"
+)
+
+type labelRequirementCreator interface {
+	createLabelRequirement() ([]labels.Requirement, error)
+}
+
+type masterLabel struct {
+	op selection.Operator
+}
+
+func (ml masterLabel) createLabelRequirement() ([]labels.Requirement, error) {
+	requirement, err := labels.NewRequirement(values.LabelMaster, ml.op, nil)
+	if err != nil {
+		log.Error(err, "unable to create requirement "+values.LabelMaster)
+		return []labels.Requirement{}, err
+	}
+	return []labels.Requirement{*requirement}, nil
+}
+
+type opsLabel struct {
+	label string
+	op    selection.Operator
+}
+
+func (ol opsLabel) createLabelRequirement() ([]labels.Requirement, error) {
+	requirement, err := labels.NewRequirement(ol.label, ol.op, nil)
+	if err != nil {
+		log.Error(err, "unable to create requirement "+ol.label)
+		return []labels.Requirement{}, err
+	}
+	return []labels.Requirement{*requirement}, nil
+}
+
+type nodeSelectorLabel struct {
+	value string
+	op    selection.Operator
+}
+
+func (nl nodeSelectorLabel) createLabelRequirement() ([]labels.Requirement, error) {
+	if nl.value == AllNodeSelector {
+		return []labels.Requirement{}, nil
+	}
+	var requirements []labels.Requirement
+	// if nodeselector is "no-label", will get the nodes which not have label
+	if nl.value == NoNodeSelector {
+		requirement, err := labels.NewRequirement(values.LabelNodeSelector, selection.DoesNotExist, nil)
+		if err != nil {
+			log.Error(err, "unable to create requirement "+values.LabelNodeSelector)
+			return []labels.Requirement{}, err
+		}
+		requirements = append(requirements, *requirement)
+		return requirements, nil
+	}
+	requirement, err := labels.NewRequirement(values.LabelNodeSelector, nl.op, []string{nl.value})
+	if err != nil {
+		log.Error(err, "unable to create requirement "+values.LabelNodeSelector)
+		return []labels.Requirement{}, err
+	}
+	requirements = append(requirements, *requirement)
+	return requirements, nil
+}
+
+type serialLabel struct {
+	op selection.Operator
+}
+
+func (sl serialLabel) createLabelRequirement() ([]labels.Requirement, error) {
+	requirement, err := labels.NewRequirement(values.LabelSerial, sl.op, nil)
+	if err != nil {
+		log.Error(err, "unable to create requirement "+values.LabelSerial)
+		return []labels.Requirement{}, err
+	}
+	return []labels.Requirement{*requirement}, nil
+}
+
+func createRequirement(labelsList []labelRequirementCreator) ([]labels.Requirement, error) {
+	var requirements []labels.Requirement
+	for _, label := range labelsList {
+		requirement, err := label.createLabelRequirement()
+		if err != nil {
+			log.Error(err, "unable to create requirement "+values.LabelNodeSelector)
+			return []labels.Requirement{}, err
+		}
+		requirements = append(requirements, requirement...)
+	}
+	return requirements, nil
+}
+
+type nodeRequirementCreator interface {
+	createNodeRequirement(ctx context.Context, r common.ReadStatusWriter, nodeSelector string) ([]labels.Requirement, error)
+}
+
+type commonsNodesRequirement struct {
+	nodeSelector string
+	op           selection.Operator
+}
+
+func newCommonsNodesRequirement(nodeSelector string, op selection.Operator) commonsNodesRequirement {
+	return commonsNodesRequirement{
+		nodeSelector: nodeSelector,
+		op:           op,
+	}
+}
+
+func (c commonsNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) {
+	labelList := []labelRequirementCreator{
+		masterLabel{op: selection.DoesNotExist},
+		nodeSelectorLabel{value: c.nodeSelector, op: c.op},
+	}
+	requirements, err := createRequirement(labelList)
+	if err != nil {
+		return []labels.Requirement{}, err
+	}
+	return requirements, nil
+}
+
+type opsNodesRequirement struct {
+	common commonsNodesRequirement
+	ops    opsLabel
+}
+
+func newopsNodesRequirement(nodeSelector string, nodeSelectorOp selection.Operator, ops opsLabel) opsNodesRequirement {
+	return opsNodesRequirement{
+		common: newCommonsNodesRequirement(nodeSelector, nodeSelectorOp),
+		ops:    ops,
+	}
+}
+
+func (o opsNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) {
+	labelList := []labelRequirementCreator{
+		o.ops,
+	}
+	requirements, err := createRequirement(labelList)
+	if err != nil {
+		return []labels.Requirement{}, err
+	}
+	commonRequirements, err := o.common.createNodeRequirement(ctx, r)
+	if err != nil {
+		return []labels.Requirement{}, err
+	}
+	requirements = append(requirements, commonRequirements...)
+	return requirements, nil
+}
+
+type serialNodesRequirement struct {
+	common   commonsNodesRequirement
+	serialOp selection.Operator
+}
+
+func newSerialNodesRequirement(nodeSelector string, nodeSelectorOp selection.Operator, serialrOp selection.Operator) serialNodesRequirement {
+	return serialNodesRequirement{
+		common:   newCommonsNodesRequirement(nodeSelector, nodeSelectorOp),
+		serialOp: serialrOp,
+	}
+}
+
+func (o serialNodesRequirement) createNodeRequirement(ctx context.Context, r common.ReadStatusWriter) ([]labels.Requirement, error) {
+	labelList := []labelRequirementCreator{
+		serialLabel{op: o.serialOp},
+	}
+	requirements, err := createRequirement(labelList)
+	if err != nil {
+		return []labels.Requirement{}, err
+	}
+	commonRequirements, err := o.common.createNodeRequirement(ctx, r)
+	if err != nil {
+		return []labels.Requirement{}, err
+	}
+	requirements = append(requirements, commonRequirements...)
+	return requirements, nil
+}
diff --git a/cmd/operator/controllers/times.go b/cmd/operator/controllers/times.go
new file mode 100644
index 0000000000000000000000000000000000000000..f651c0e42fea72b979b51d5e5e8f9378af0c88ad
--- /dev/null
+++ b/cmd/operator/controllers/times.go
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) Huawei Technologies Co., Ltd. 2024. All rights reserved.
+ * KubeOS is licensed under the Mulan PSL v2.
+ * You can use this software according to the terms and conditions of the Mulan PSL v2.
+ * You may obtain a copy of Mulan PSL v2 at:
+ *     http://license.coscl.org.cn/MulanPSL2
+ * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+ * PURPOSE.
+ * See the Mulan PSL v2 for more details.
+ */
+
+// Package controllers contains the Reconcile of operator
+
+package controllers
+
+import (
+	"fmt"
+	"regexp"
+	"time"
+)
+
+const (
+	DATE_TIME             = "2006-01-02 15:04:05"
+	TIME_ONLY             = "15:04:05"
+	ExecutionModeSerial   = "serial"
+	ExecutionModeParallel = "parallel"
+	oneDayTime            = 24 * time.Hour
+)
+
+func isWithinTimeWindow(start, end string) (bool, error) {
+	if start == "" && end == "" {
+		return true, nil
+	}
+	if start == "" || end == "" {
+		return false, fmt.Errorf("invalid TimeWindow: The start time and end time must be both empty or not empty")
+	}
+	layoutStart, err := checkTimeValid(start)
+	if err != nil {
+		return false, err
+	}
+	layoutEnd, err := checkTimeValid(end)
+	if err != nil {
+		return false, err
+	}
+	if layoutStart != layoutEnd {
+		return false, fmt.Errorf("invalid TimeWindow: Start Time should have same time format with End Time")
+	}
+	now := time.Now()
+	timeFormat := now.Format(layoutStart)
+	now, err = time.ParseInLocation(layoutStart, timeFormat, now.Location())
+	startTime, err := time.ParseInLocation(layoutStart, start, now.Location())
+	if err != nil {
+		return false, err
+	}
+	endTime, err := time.ParseInLocation(layoutStart, end, now.Location())
+	if err != nil {
+		return false, err
+	}
+	if endTime.Equal(startTime) {
+		return false, fmt.Errorf("invalid TimeWindow: start time is equal to end time")
+	}
+	if endTime.Before(startTime) {
+		if layoutStart == DATE_TIME {
+			return false, fmt.Errorf("invalid TimeWindow: start time %s is after end time %s",
+				startTime.Format(layoutStart), endTime.Format(layoutEnd))
+		}
+		endTime = endTime.Add(oneDayTime)
+		if now.Before(startTime) {
+			now = now.Add(oneDayTime)
+		}
+
+	}
+	return now.After(startTime) && now.Before(endTime), nil
+}
+
+func checkTimeValid(checkTime string) (string, error) {
+	reDateTime, err := regexp.Compile("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$")
+	if err != nil {
+		return "", err
+	}
+	reTimeOnly, err := regexp.Compile("^\\d{2}:\\d{2}:\\d{2}$")
+	if err != nil {
+		return "", err
+	}
+
+	if reDateTime.MatchString(checkTime) {
+		_, err := time.Parse(DATE_TIME, checkTime)
+		if err != nil {
+			return "", err
+		}
+		return DATE_TIME, nil
+
+	}
+	if reTimeOnly.MatchString(checkTime) {
+		_, err := time.Parse(TIME_ONLY, checkTime)
+		if err != nil {
+			return "", err
+		}
+		return TIME_ONLY, nil
+
+	}
+	return "", fmt.Errorf("invalid TimeWindow: invalid date format, please use date format YYYY-MM-DD HH:MM:SS, or only HH:MM:SS")
+}
diff --git a/cmd/operator/main.go b/cmd/operator/main.go
index 6b90b26b15f0eb1e8998052e6d29bd019cb091f1..79b23ec7e312adfbf0d9ce7c41c5462cec1b375a 100644
--- a/cmd/operator/main.go
+++ b/cmd/operator/main.go
@@ -14,6 +14,7 @@ package main
 
 import (
 	"os"
+	"strings"
 	"time"
 
 	zaplogfmt "github.com/sykesm/zap-logfmt"
@@ -38,6 +39,11 @@ var (
 	setupLog = ctrl.Log.WithName("setup")
 )
 
+const (
+	DEBUGLOGLEVEL = "debug"
+	INFOLOGLEVEL  = "info"
+)
+
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 
@@ -51,7 +57,12 @@ func main() {
 		encoder.AppendString(ts.UTC().Format(time.RFC3339Nano))
 	}
 	logfmtEncoder := zaplogfmt.NewEncoder(configLog)
-	logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder))
+	level := zap.Level(zapcore.Level(0))
+	goLog := strings.ToLower(os.Getenv("GO_LOG"))
+	if goLog == DEBUGLOGLEVEL {
+		level = zap.Level(zapcore.Level(-1))
+	}
+	logger := zap.New(zap.UseDevMode(true), zap.WriteTo(os.Stdout), zap.Encoder(logfmtEncoder), level)
 	ctrl.SetLogger(logger)
 
 	mgr, err := common.NewControllerManager(setupLog, scheme)
diff --git a/docs/example/config/admin-container/admin-container.yaml b/docs/example/config/admin-container/admin-container.yaml
index b5ec9e03061ee740141862b981a87d508d6a30a5..518b955fb64eb7397e85027ab440b1935d42a018 100644
--- a/docs/example/config/admin-container/admin-container.yaml
+++ b/docs/example/config/admin-container/admin-container.yaml
@@ -1,63 +1,63 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: root-secret
-data:
-  ssh-pub-key: your-ssh-pub-key
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: admin-container-sysmaster
-  namespace: default
-  labels:
-      control-plane: admin-container-sysmaster
-spec:
-  selector:
-    matchLabels:
-      control-plane: admin-container-sysmaster
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        control-plane: admin-container-sysmaster
-    spec:
-      hostPID: true
-      containers:
-      - name: admin-container-sysmaster
-        image: your_imageRepository/admin_imageName:version
-        imagePullPolicy: Always
-        securityContext:
-          privileged: true
-        ports:
-          - containerPort: 22
-        # sysmaster要求
-        env:
-          - name: container
-            value: containerd
-        volumeMounts:
-        # name 必须与下面的卷名匹配
-        - name: secret-volume
-        # mountPath必须为/etc/secret-volume
-          mountPath: /etc/secret-volume
-          readOnly: true
-      nodeName: your-worker-node-name
-      volumes:
-        - name: secret-volume
-          secret:
-            # secretName必须与上面指定的Secret的name相同
-            secretName: root-secret
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: admin-container-sysmaster
-  namespace: default
-spec:
-  type: NodePort
-  ports:
-    - port: 22
-      targetPort: 22
-      nodePort: your-exposed-port
-  selector:
-      control-plane: admin-container-sysmaster
\ No newline at end of file
+apiVersion: v1
+kind: Secret
+metadata:
+  name: root-secret
+data:
+  ssh-pub-key: <your-ssh-pub-key-encoded-with-base64>
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: admin-container-sysmaster
+  namespace: default
+  labels:
+    control-plane: admin-container-sysmaster
+spec:
+  selector:
+    matchLabels:
+      control-plane: admin-container-sysmaster
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        control-plane: admin-container-sysmaster
+    spec:
+      hostPID: true
+      containers:
+        - name: admin-container-sysmaster
+          image: <your_imageRepository/admin_imageName:version>
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          ports:
+            - containerPort: 22
+          # sysmaster要求
+          env:
+            - name: container
+              value: containerd
+          volumeMounts:
+            # name 必须与下面的卷名匹配
+            - name: secret-volume
+              # mountPath必须为/etc/secret-volume
+              mountPath: /etc/secret-volume
+              readOnly: true
+      nodeName: <your-worker-node-name>
+      volumes:
+        - name: secret-volume
+          secret:
+            # secretName必须与上面指定的Secret的name相同
+            secretName: root-secret
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: admin-container-sysmaster
+  namespace: default
+spec:
+  type: NodePort
+  ports:
+    - port: 22
+      targetPort: 22
+      nodePort: <your-exposed-port>
+  selector:
+    control-plane: admin-container-sysmaster
diff --git a/docs/example/config/crd/upgrade.openeuler.org_os.yaml b/docs/example/config/crd/upgrade.openeuler.org_os.yaml
index 98ef1f5645df899bb3f8e721f221eaf2b31f934f..8a29e0c37d037292727f4039a018168e2f0516fc 100644
--- a/docs/example/config/crd/upgrade.openeuler.org_os.yaml
+++ b/docs/example/config/crd/upgrade.openeuler.org_os.yaml
@@ -18,7 +18,7 @@ spec:
   versions:
   - name: v1alpha1
     additionalPrinterColumns:
-    - name: OSVersion
+    - name: OS VERSION
       jsonPath: .spec.osversion
       type: string
       description: The version of OS
@@ -57,9 +57,19 @@ spec:
                 type: string
               evictpodforce:
                 type: boolean
+              executionmode:
+                default: parallel
+                enum:
+                - serial
+                - parallel
+                type: string
               flagSafe:
                 type: boolean
               imagetype:
+                enum:
+                - docker
+                - disk
+                - containerd
                 type: string
               imageurl:
                 type: string
@@ -68,8 +78,13 @@ spec:
               mtls:
                 type: boolean
               nodeselector:
+                default: no-label
                 type: string
               opstype:
+                enum:
+                - upgrade
+                - config
+                - rollback
                 type: string
               osversion:
                 type: string
@@ -101,6 +116,19 @@ spec:
                   version:
                     type: string
                 type: object
+              timeinterval:
+                default: 15
+                type: integer
+              timewindow:
+                properties:
+                  endtime:
+                    type: string
+                  starttime:
+                    type: string
+                required:
+                - endtime
+                - starttime
+                type: object
               upgradeconfigs:
                 description: SysConfigs defines all configurations expected by the user
                 properties:
diff --git a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml
index a7bad3f075066ad1296f263020970e9fbda91b2c..47de87e3b7ef071ee231d4b76934a1688c3b0a98 100644
--- a/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml
+++ b/docs/example/config/crd/upgrade.openeuler.org_osinstances.yaml
@@ -22,19 +22,19 @@ spec:
       type: string
       jsonPath: .spec.nodestatus
       description: The status of node
-    - name: SYSCONFIG STATUS
+    - name: SYSCONFIG-VERSION-CURRENT
       type: string
       jsonPath: .status.sysconfigs.version
       description: The current status of sysconfig
-    - name: SYSCONFIG SPEC
+    - name: SYSCONFIG-VERSION-DESIRED
       type: string
       jsonPath: .spec.sysconfigs.version
       description: The expected version of sysconfig
-    - name: UPGRADECONFIG STATUS
+    - name: UPGRADECONFIG-VERSION-CURRENT
       type: string
       jsonPath: .status.upgradeconfigs.version
       description: The current version of upgradeconfig
-    - name: UPGRADECONFIG SPEC
+    - name: UPGRADECONFIG-VERSION-DESIRED
       type: string
       jsonPath: .spec.upgradeconfigs.version
       description: The expected version of upgradeconfig
@@ -53,6 +53,14 @@ spec:
           spec:
             description: OSInstanceSpec defines desired state of OS
             properties:
+              namespacedname:
+                description: NamespacedName defines name and namespace of os corresponding to osinstance
+                properties:
+                  name:
+                    type: string
+                  namespace:
+                    type: string
+                type: object
               nodestatus:
                 type: string
               sysconfigs:
diff --git a/docs/example/config/manager/manager.yaml b/docs/example/config/manager/manager.yaml
index 93d152209506e727b2a0eb61703f454325e1e5af..c35d901079d281b90c88febd7b880cab874fb411 100644
--- a/docs/example/config/manager/manager.yaml
+++ b/docs/example/config/manager/manager.yaml
@@ -20,19 +20,19 @@ spec:
         control-plane: upgrade-proxy
     spec:
       containers:
-       - name: proxy
-         command:
-          - /proxy
-         image: edit.proxy.image.addr
-         volumeMounts:
-          - name: upgrade-agent
-            mountPath: /var/run/os-agent
-         env:
-          - name: NODE_NAME
-            valueFrom:
-             fieldRef:
-              apiVersion: v1
-              fieldPath: spec.nodeName
+        - name: proxy
+          command:
+            - /proxy
+          image: <edit.proxy.image.addr>
+          volumeMounts:
+            - name: upgrade-agent
+              mountPath: /var/run/os-agent
+          env:
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
       volumes:
         - name: upgrade-agent
           hostPath:
@@ -56,24 +56,33 @@ spec:
         control-plane: upgrade-operator
     spec:
       containers:
-      - command:
-        - /operator
-        image: edit.operator.image.addr
-        name: operator
-        securityContext:
-          allowPrivilegeEscalation: false
-          runAsUser: 6552
-          runAsGroup: 6552
-        resources:
-          limits:
-            cpu: 100m
-            memory: 30Mi
-          requests:
-            cpu: 100m
-            memory: 20Mi
+        - command:
+            - /operator
+          image: <edit.operator.image.addr>
+          name: operator
+          volumeMounts:
+            - name: date-config
+              mountPath: /etc/localtime
+          securityContext:
+            allowPrivilegeEscalation: false
+            runAsUser: 6552
+            runAsGroup: 6552
+          resources:
+            limits:
+              cpu: 100m
+              memory: 30Mi
+            requests:
+              cpu: 100m
+              memory: 20Mi
+      volumes:
+      - name: date-config
+        hostPath: 
+          path: /etc/localtime        
       terminationGracePeriodSeconds: 10
       nodeSelector:
         node-role.kubernetes.io/control-plane: ""
       tolerations:
         - key: "node-role.kubernetes.io/master"
           operator: "Exists"
+        - key: "node-role.kubernetes.io/control-plane"
+          operator: "Exists"
diff --git a/docs/example/config/samples/upgrade_v1alpha1_os.yaml b/docs/example/config/samples/upgrade_v1alpha1_os.yaml
index ca3da1b2c9a52651f458f33bbafca5435ae2f970..d75288310a0a936bcf6c1949af47d7e5293db3f7 100644
--- a/docs/example/config/samples/upgrade_v1alpha1_os.yaml
+++ b/docs/example/config/samples/upgrade_v1alpha1_os.yaml
@@ -1,39 +1,38 @@
 apiVersion: upgrade.openeuler.org/v1alpha1
 kind: OS
 metadata:
-    name: os-sample
+  name: os-sample
 spec:
-    imagetype: docker/containerd/disk
-    opstype: upgrade/config/rollback
-    osversion: edit.os.version
-    maxunavailable: edit.node.upgrade.number
-    containerimage: ""
-    evictpodforce: true
-    imageurl: ""
-    checksum: image digests
-    flagSafe: false
-    mtls: false
-    nodeselector: edit.nodes.label
-    sysconfigs:
-        version: edit.sysconfig.version
-        configs:
-            - model: kernel.systcl
-            contents:
-                - key: kernel param key1
-                  value: kernel param value1
-                - key: kernel param key2
-                  value: kernel param value2
-            - model: kernel.systcl.persist
-              configpath: persist file path
-              contents:
-                - key: kernel param key3
-                  value: kernel param value3
-                  operation: delete
-    upgradeconfigs:
-        version: edit.upgradeconfig.version
-        configs:
-            - model: kernel.systcl
-            contents:
-                - key: kernel param key4
-                  value: kernel param value4
-                  operation: delete
+  imagetype: docker/containerd/disk
+  opstype: upgrade/config/rollback
+  osversion: edit.os.version
+  maxunavailable: edit.node.upgrade.number
+  containerimage: ""
+  evictpodforce: true
+  imageurl: ""
+  checksum: image digests
+  flagSafe: false
+  mtls: false
+  sysconfigs:
+    version: edit.sysconfig.version
+    configs:
+      - model: kernel.sysctl
+        contents:
+          - key: kernel param key1
+            value: kernel param value1
+          - key: kernel param key2
+            value: kernel param value2
+      - model: kernel.sysctl.persist
+        configpath: persist file path
+        contents:
+          - key: kernel param key3
+            value: kernel param value3
+            operation: delete
+  upgradeconfigs:
+    version: edit.upgradeconfig.version
+    configs:
+      - model: kernel.sysctl
+        contents:
+          - key: kernel param key4
+            value: kernel param value4
+            operation: delete
diff --git a/docs/quick-start.md b/docs/quick-start.md
index 13bb08d9e3229b457204950ce0ac9589dd5da4f8..1d59048ebd1bb9b245937278f922cc2346bf386e 100644
--- a/docs/quick-start.md
+++ b/docs/quick-start.md
@@ -1,18 +1,22 @@
 # 快速使用指导
 
-## 编译及部署
+[TOC]
 
-### 编译指导
+## 编译指导
 
 * 编译环境：openEuler Linux x86/AArch64
+
 * 进行编译需要以下包：
   * golang(大于等于1.15版本)
   * make
   * git
+  * rust(大于等于1.64版本)
+  * cargo(大于等于1.64版本)
+  * openssl-devel
 
   ``` shell
-  sudo yum install golang make git
-  ```  
+  sudo yum install golang make git rust cargo openssl-devel
+  ```
 
 * 使用git获取本项目的源码
 
@@ -27,77 +31,128 @@
 
   ```shell
   cd KubeOS
-  sudo make 
-  ```  
-
-  * proxy及operator容器镜像构建
-    * proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕
-    * 请用户自行编写Dockerfile来构建镜像，请注意
-      * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性
-      * 需将operator和proxy拷贝到baseimage上
-      * 请确保proxy属主和属组为root，文件权限为500
-      * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500
-      * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应
-    * 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库
-    * Dockerfile参考如下, Dockerfile也可以使用多阶段构建:
-
-      ``` dockerfile
-      FROM your_baseimage
-      COPY ./bin/proxy /proxy
-      ENTRYPOINT ["/proxy"]
-      FROM your_baseimage
-      COPY --chown=6552:6552 ./bin/operator /operator
-      ENTRYPOINT ["/operator"]
-      ```
+  sudo make
+  # 编译生成的二进制在bin目录下，查看二进制
+  tree bin
+  bin
+  ├── operator
+  ├── os-agent
+  ├── proxy
+  ├── rust
+  │   ├── ...
+  │   └── release
+  │       ├── ...
+  │       ├── os-agent
+  │       └── proxy
+  ```
 
-      ```shell
-      # 指定proxy的镜像仓库，镜像名及版本
-      export IMG_PROXY=your_imageRepository/proxy_imageName:version
-      # 指定proxy的Dockerfile地址
-      export DOCKERFILE_PROXY=your_dockerfile_proxy
-      # 指定operator的镜像仓库，镜像名及版本
-      export IMG_OPERATOR=your_imageRepository/operator_imageName:version
-      # 指定operator的Dockerfile路径
-      export DOCKERFILE_OPERATOR=your_dockerfile_operator
-
-      # 镜像构建
-      docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} .
-      docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} .
-      # 推送镜像到镜像仓库
-      docker push ${IMG_OPERATOR}
-      docker push ${IMG_PROXY}
-      ```
+  * ```bin/proxy```、```bin/os-agent```为go语言编写的proxy和os-agent，```bin/rust/release/proxy```、```bin/rust/release/os-agent```为rust语言编写的proxy和os-agent，二者功能一致。
+
+## 镜像构建指导
+
+### proxy及operator镜像构建指导
+
+* proxy及operator容器镜像构建使用docker，请先确保docker已经安装和配置完毕
+
+* 请用户自行编写Dockerfile来构建镜像，请注意
+  * operator和proxy需要基于baseimage进行构建，用户保证baseimage的安全性
+  * 需将operator和proxy拷贝到baseimage上
+  * 请确保proxy属主和属组为root，文件权限为500
+  * 请确保operator属主和属组为在容器内运行operator的用户，文件权限为500
+  * operator和proxy的在容器内的位置和容器启动时运行的命令需与部署operator的yaml中指定的字段相对应
+
+* 首先指定镜像仓库地址、镜像名及版本，Dockerfile路径，然后构建并推送镜像到镜像仓库
+
+* Dockerfile参考如下, Dockerfile也可以使用多阶段构建:
+
+  `proxy`容器镜像Dockerfile
+
+  ``` dockerfile
+  FROM openeuler/openeuler:24.03-lts
+  COPY ./bin/proxy /proxy
+  ENTRYPOINT ["/proxy"]
+  ```
+
+  `operator`容器镜像Dockerfile
+
+  ``` dockerfile
+  FROM openeuler/openeuler:24.03-lts
+  COPY --chown=6552:6552 ./bin/operator /operator
+  ENTRYPOINT ["/operator"]
+  ```
 
-* OS虚拟机镜像制作
-  * 制作注意事项
-    * 请确保已安装qemu-img，bc，parted，tar，yum，docker, dosfstools
-    * 容器OS镜像制作需要使用root权限
-    * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库
-    * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式
-    * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间
-    * 容器镜像制作时不支持用户自定义配置挂载文件
-    * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。
-    * 请确保os-agent属主和属组为root，建议os-agent文件权限为500
-      * 容器OS虚拟机镜像制作
-          进入scripts目录，执行脚本
-
-          ```shell
-          cd scripts
-          bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0'''
-          ```
-
-        * 其中 xx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。
-        * 示例命令使用的密码为```openEuler12#$```
-        * 容器 OS 镜像制作完成后，会在 scripts 目录下生成：
-          * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。
-          * qcow2 格式的系统镜像 system.qcow2。
-          * 可用于升级的根文件系统分区镜像 update.img 。
-        * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景，x86 架构的虚拟机使用 legacy 启动模式启动需制作镜像时指定-l参数
-        * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool)
-        * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。
-  * 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。
-
-### 部署指导
+  ```shell
+  # 指定proxy的镜像仓库，镜像名及版本
+  export IMG_PROXY=your_imageRepository/proxy_imageName:version
+  # 指定proxy的Dockerfile地址
+  export DOCKERFILE_PROXY=your_dockerfile_proxy
+  # 指定operator的镜像仓库，镜像名及版本
+  export IMG_OPERATOR=your_imageRepository/operator_imageName:version
+  # 指定operator的Dockerfile路径
+  export DOCKERFILE_OPERATOR=your_dockerfile_operator
+  
+  # 镜像构建
+  docker build -t ${IMG_OPERATOR} -f ${DOCKERFILE_OPERATOR} .
+  docker build -t ${IMG_PROXY} -f ${DOCKERFILE_PROXY} .
+  # 推送镜像到镜像仓库
+  docker push ${IMG_OPERATOR}
+  docker push ${IMG_PROXY}
+  ```
+
+### KubeOS虚拟机镜像制作指导
+
+* 制作注意事项
+  * 请确保已安装qemu-img，bc，parted，tar，yum，docker，dosfstools
+  * 容器OS镜像制作需要使用root权限
+  * 容器OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库
+  * 容器OS镜像制作之前需要先将当前机器上的selinux关闭或者设为允许模式
+  * 使用默认rpmlist进行容器OS镜像制作出来的镜像默认和制作工具保存在相同路径，该分区至少有25G的剩余空间
+  * 容器镜像制作时不支持用户自定义配置挂载文件
+  * 容器OS镜像制作工具执行异常中断，可能会残留文件、目录或挂载，需用户手动清理，对于可能残留的rootfs目录，该目录虽然权限为555，但容器OS镜像制作在开发环境进行，不会对生产环境产生影响。
+  * 请确保os-agent属主和属组为root，建议os-agent文件权限为500
+
+* 容器OS虚拟机镜像制作
+    进入scripts目录，执行脚本
+
+    ```shell
+    cd scripts
+    bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0'''
+    ```
+
+    参数说明如下:
+
+    ```bash
+    Usage : kbimg create vm-image -p iso-path -v os-version -b os-agent-dir -e os-password
+          or
+            kbimg create vm-image -d repository/name:tag
+
+    options:
+        -p                       repo path
+        -v                       KubeOS version
+        -b                       path of os-agent binary
+        -e                       os encrypted password
+        -d                       docker image like repository/name:tag
+        -l                       boot to legacy BIOS mode, if not specify, then UEFI mode
+    -h,--help                show help information
+    ```
+
+  * 其中 xxx.repo 为制作镜像所需要的 yum 源，yum 源建议配置为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。
+  * 容器 OS 镜像制作完成后，会在 scripts 目录下生成：
+    * raw格式的系统镜像system.img，system.img大小默认为20G，支持的根文件系统分区大小<2020MiB，持久化分区<16GB。
+    * qcow2 格式的系统镜像 system.qcow2。
+    * 可用于升级的根文件系统分区镜像 update.img 。
+  * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机场景。若x86 架构的虚拟机需要使用 legacy 启动模式，需制作镜像时指定-l参数
+  * 默认root密码为openEuler12#$
+  * 您可通过`openssl passwd -6 -salt $(head -c18 /dev/urandom | openssl base64)`命令生成root密码并通过`-e`参数配置密码
+  * 容器OS运行底噪<150M (不包含k8s组件及相关依赖kubernetes-kubeadm，kubernetes-kubelet， containernetworking-plugins，socat，conntrack-tools，ebtables，ethtool)
+  * 本项目不提供容器OS镜像，仅提供裁剪工具，裁剪出来的容器OS内部的安全性由OS发行商保证。
+  * 详细参数说明请见[《容器OS镜像制作指导》](../docs/user_guide/%E5%AE%B9%E5%99%A8OS%E9%95%9C%E5%83%8F%E5%88%B6%E4%BD%9C%E6%8C%87%E5%AF%BC.md)
+
+* 声明： os-agent使用本地unix socket进行通信，因此不会新增端口。下载镜像的时候会新增一个客户端的随机端口，1024~65535使用完后关闭。proxy和operator与api-server通信时作为客户端也会有一个随机端口，基于kubernetes的operator框架，必须使用端口。他们部署在容器里。
+
+## 部署指导
+
+### os-operator和os-proxy部署指导
 
 * 环境要求
   * openEuler Linux x86/AArch64系统
@@ -143,18 +198,35 @@
     kubectl get pods -A
     ```
 
-### 使用指导
-
-#### 注意事项
-
-* 容器OS升级为所有软件包原子升级，默认不在容器OS内提供单包升级能力。
-* 容器OS升级为双区升级的方式，不支持更多分区数量。
-* 单节点的升级过程的日志可在节点的/var/log/message文件查看。
-* 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。
-* 使用docker镜像升级和mtls双向认证仅支持 openEuler 22.09 及之后的版本
-* 不支持跨大版本升级
-
-#### 参数说明
+## 使用指导
+
+### 注意事项
+
+* 公共注意事项
+  * 仅支持虚拟机x86和arm64 UEFI场景。
+  * 当前不支持集群节点OS多版本管理，即集群中OS的CR只能为一个。
+  * 使用kubectl apply通过YAML创建或更新OS的CR时，不建议并发apply，当并发请求过多时，kube-apiserver会无法处理请求导致失败。
+  * 如用户配置了容器镜像仓的证书或密钥，请用户保证证书或密钥文件的权限最小。
+* 升级注意事项
+  * 升级为所有软件包原子升级，默认不提供单包升级能力。
+  * 升级为双区升级的方式，不支持更多分区数量。
+  * 当前暂不支持跨大版本升级。
+  * 单节点的升级过程的日志可在节点的 /var/log/messages 文件查看。
+  * 请严格按照提供的升级和回退流程进行操作，异常调用顺序可能会导致系统无法升级或回退。
+  * 节点上containerd如需配置ctr使用的私有镜像，请将配置文件host.toml按照ctr指导放在/etc/containerd/certs.d目录下。
+
+* 配置注意事项
+  * 用户自行指定配置内容，用户需保证配置内容安全可靠 ，尤其是持久化配置（kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next），KubeOS不对参数有效性进行检验。
+  * opstype=config时，若osversion与当前集群节点的OS版本不一致，配置不会进行。
+  * 当前仅支持kernel参数临时配置（kernel.sysctl）、持久化配置（kernel.sysctl.persist）和grub cmdline配置（grub.cmdline.current和grub.cmdline.next）。
+  * 持久化配置会写入persist持久化分区，升级重启后配置保留；kernel参数临时配置重启后不保留。
+  * 配置grub.cmdline.current或grub.cmdline.next时，如为单个参数（非key=value格式参数），请指定key为该参数，value为空。
+  * 进行配置删除（operation=delete）时，key=value形式的配置需保证key、value和实际配置一致。
+  * 配置不支持回退，如需回退，请修改配置版本和配置内容，重新下发配置。
+  * 配置出现错误，节点状态陷入config时，请将配置版本恢复成上一版本并重新下发配置，从而使节点恢复至idel状态。 但是请注意：出现错误前已经配置完成的参数无法恢复。
+  * 在配置grub.cmdline.current或grub.cmdline.next时，若需要将已存在的“key=value”格式的参数更新为只有key无value格式，比如将“rd.info=0”更新成rd.info，需要先删除“key=value”，然后在下一次配置时，添加key。不支持直接更新或者更新删除动作在同一次完成。
+
+#### OS CR参数说明
 
 在集群中创建类别为OS的定制对象，设置相应字段。类别OS来自于安装和部署章节创建的CRD对象，字段及说明如下：
 
@@ -164,23 +236,22 @@
 
   | 参数            |参数类型  | 参数说明                                                     | 使用说明 | 是否必选         |
   | -------------- | ------ | ------------------------------------------------------------ | ----- | ---------------- |
-  | imagetype      | string | 使用的升级镜像的类型           | 需为 docker ，containerd ，或者是 disk，其他值无效，且该参数仅在升级场景有效。<br> **注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。|是               |
-  | opstype        | string | 进行的操作，升级,回退或者配置 | 需为 upgrade ，config 或者 rollback ，其他值无效 |是               |
-  | osversion      | string | 用于升级或回退的镜像的OS版本          | 需为 KubeOS version , 例如: KubeOS 1.0.0|是               |
-  | maxunavailable | int    | 同时进行升级或回退的节点数 | maxunavailable值设置为大于实际集群的节点数时也可正常部署，升级或回退时会按照集群内实际节点数进行|是               |
-  | containerimage    | string | 用于升级的容器镜像               | 需要为容器镜像格式：[REPOSITORY/NAME[:TAG@DIGEST]](https://docs.docker.com/engine/reference/commandline/tag/#extended-description)，仅在使用容器镜像升级场景下有效|是               |
-  | imageurl       | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如：<https://192.168.122.15/update.img> 仅在使用磁盘镜像升级场景下有效|是               |
+  | imagetype      | string | 升级镜像的类型           | 仅支持docker ，containerd ，或者是 disk，仅在升级场景有效。**注意**：若使用containerd，agent优先使用crictl工具拉取镜像，没有crictl时才会使用ctr命令拉取镜像。使用ctr拉取镜像时，镜像如果在私有仓内，需按照[官方文档](https://github.com/containerd/containerd/blob/main/docs/hosts.md)在/etc/containerd/certs.d目录下配置私有仓主机信息，才能成功拉取镜像。 |是               |
+  | opstype        | string | 操作类型：升级,回退或者配置 | 仅支持upgrade ，config 或者 rollback |是               |
+  | osversion      | string | 升级/回退的目标版本  | osversion需与节点的目标os版本对应（节点上/etc/os-release中PRETTY_NAME字段或k8s检查到的节点os版本） 例如：KubeOS 1.0.0。 |是               |
+  | maxunavailable | int    | 每批同时进行升级/回退/配置的节点数。 | maxunavailable值大于实际节点数时，取实际节点数进行升级/回退/配置。 |是               |
+  | containerimage    | string | 用于升级的容器镜像               | 仅在imagetype是容器类型时生效，仅支持以下3种格式的容器镜像地址： repository/name repository/name@sha256:xxxx repository/name:tag |是               |
+  | imageurl       | string | 用于升级的磁盘镜像的地址 | imageurl中包含协议，只支持http或https协议，例如：<https://192.168.122.15/update.img> ，仅在使用磁盘镜像升级场景下有效 |是               |
   | checksum       | string | 用于升级的磁盘镜像校验的checksum(SHA-256)值或者是用于升级的容器镜像的digests值                      | 仅在升级场景下有效 |是               |
   | flagSafe       | bool   | 当imageurl的地址使用http协议表示是否是安全的                 | 需为 true 或者 false ，仅在imageurl使用http协议时有效 |是               |
   | mtls           | bool   | 用于表示与imageurl连接是否采用https双向认证     | 需为 true 或者 false ，仅在imageurl使用https协议时有效|是               |
   | cacert         | string | https或者https双向认证时使用的根证书文件                       | 仅在imageurl使用https协议时有效| imageurl使用https协议时必选 |
   | clientcert     | string | https双向认证时使用的客户端证书文件                          | 仅在使用https双向认证时有效|mtls为true时必选 |
   | clientkey      | string | https双向认证时使用的客户端公钥                              | 仅在使用https双向认证时有效|mtls为true时必选 |
-  | evictpodforce      | bool | 用于表示升级/回退时是否强制驱逐pod                            | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 |
+  | evictpodforce      | bool | 升级/回退时是否强制驱逐pod                            | 需为 true 或者 false ，仅在升级或者回退时有效| 必选 |
+  | sysconfigs      | / | 配置设置                          | 1. “opstype=config”时只进行配置。  2.“opstype=upgrade/rollback”时，代表升级/回退后配置，即在升级/回退重启后进行配置。```配置（Settings）指导``` | “opstype=config”时必选 |
+  | upgradeconfigs | / | 升级前配置设置                       | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 |
   | nodeselector      | string | 需要进行升级/配置/回滚操作的节点label                           | 用于只对具有某些特定label的节点而不是集群所有worker节点进行运维的场景，需要进行运维操作的节点需要包含key为upgrade.openeuler.org/node-selector的label，nodeselector为该label的value值，此参数不配置时，或者配置为""时默认对所有节点进行操作| 可选 |
-  | sysconfigs      | / | 需要进行配置的参数值                            | 在配置或者升级或者回退机器时有效，在升级或者回退操作之后即机器重启之后起效，详细字段说明请见```配置（Settings）指导```| 可选 |
-  | upgradeconfigs      | / | 需要升级前进行的配置的参数值                            | 在升级或者回退时有效，在升级或者回退操作之前起效，详细字段说明请见```配置（Settings）指导```| 可选 |  
-
 #### 升级指导
 
 * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如下，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml;
@@ -343,12 +414,13 @@
     kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage'
     ```
 
-* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的 imageurl, osversion, checksum, maxunavailable, flagSafe 或者containerimage字段进行相应修改。
+* 如果后续需要再次升级，与上面相同，对upgrade_v1alpha1_os.yaml的相应字段进行修改
 
 #### 配置（Settings）指导
 
 * Settings参数说明:
-  以进行配置时的示例yaml为例对配置的参数进行说明，示例yaml如下：
+
+  基于示例YAML对配置的参数进行说明，示例YAML如下，配置的格式（缩进）需和示例保持一致：
 
   ```yaml
   apiVersion: upgrade.openeuler.org/v1alpha1
@@ -362,15 +434,12 @@
     maxunavailable: edit.node.config.number
     containerimage: ""
     evictpodforce: false
-    imageurl: ""
     checksum: ""
-    flagSafe: false
-    mtls: false
     sysconfigs:
-        version: 1.0.0
+        version: edit.sysconfigs.version
         configs:
             - model: kernel.sysctl
-              contents:
+              contents: 
                 - key: kernel param key1
                   value: kernel param value1
                 - key: kernel param key2
@@ -380,54 +449,80 @@
               configpath: persist file path
               contents:
                 - key: kernel param key3
-                  value: kernel param value3         
+                  value: kernel param value3
+            - model: grub.cmdline.current
+              contents:
+                - key: boot param key1
+                - key: boot param key2
+                  value: boot param value2
+                - key: boot param key3
+                  value: boot param value3
+                  operation: delete
+            - model: grub.cmdline.next
+              contents:
+                - key: boot param key4
+                - key: boot param key5
+                  value: boot param value5
+                - key: boot param key6
+                  value: boot param value6
+                  operation: delete         
   ```
 
-  * 配置的参数说明如下：
-    * version: 配置的版本,通过版本差异触发配置，请修改配置后更新 version
-    * configs: 具体配置内容
-      * model: 进行的配置的类型，支持的配置类型请看[Settings 列表](#setting-列表)
-      * configpath: 如为持久化配置，配置文件路径
-      * contents: 配置参数的 key / value 和对参数的操作。
-        * key / value: 请看[Settings 列表](#setting-列表)对支持的配置的 key / value的说明。
-        * operation: 若不指定operation，则默认为添加或更新。若指定为delete，代表删除目前OS中已配置的参数。
-        **注意：** 当operation为delete时，yaml中的key/value必须和OS上想删除参数的key/value**一致**，否则删除失败。
-  * upgradeconfigs与sysconfig参数相同，upgradeconfig为升级前进行的配置，仅在升级/回滚场景起效，在升级/回滚操作执行前进行配置，只进行配置或者需要升级/回滚重启后执行配置，使用sysconfigs
+  配置的参数说明如下：
+
+  | 参数       | 参数类型 | 参数说明                    | 使用说明                                                     | 配置中是否必选          |
+  | ---------- | -------- | --------------------------- | ------------------------------------------------------------ | ----------------------- |
+  | version    | string   | 配置的版本                  | 通过version是否相等来判断配置是否触发，version为空（为""或者没有值）时同样进行判断，所以不配置sysconfigs/upgradeconfigs时，继存的version值会被清空并触发配置。 | 是                      |
+  | configs    | /        | 具体配置内容                | 包含具体配置项列表。                                         | 是                      |
+  | model      | string   | 配置的类型                  | 支持的配置类型请看附录下的```Settings列表```                 | 是                      |
+  | configpath | string   | 配置文件路径                | 仅在kernel.sysctl.persist配置类型中生效，请看附录下的```Settings列表```对配置文件路径的说明。 | 否                      |
+  | contents   | /        | 具体key/value的值及操作类型 | 包含具体配置参数列表。                                       | 是                      |
+  | key        | string   | 参数名称                    | key不能为空，不能包含"="，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中每种配置类型对key的说明。 | 是                      |
+  | value      | string   | 参数值                      | key=value形式的参数中，value不能为空，不建议配置含空格、tab键的字符串，具体请看附录下的```Settings列表```中对每种配置类型对value的说明。 | key=value形式的参数必选 |
+  | operation  | string   | 对参数进行的操作            | 仅对kernel.sysctl.persist、grub.cmdline.current、grub.cmdline.next类型的参数生效。默认为添加或更新。仅支持配置为delete，代表删除已存在的参数（key=value需完全一致才能删除）。 | 否                      |
+
+  * upgradeconfigs与sysconfigs参数相同，upgradeconfigs为升级/回退前进行的配置，仅在upgrade/rollback场景起效，sysconfigs既支持只进行配置，也支持在升级/回退重启后进行配置
+
 * 使用说明
+
   * 编写YAML文件，在集群中部署 OS 的cr实例，用于部署cr实例的YAML示例如上，假定将上面的YAML保存到upgrade_v1alpha1_os.yaml
+
   * 查看配置之前的节点的配置的版本和节点状态（NODESTATUS状态为idle）
 
     ```shell
-    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version'
+    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version'
     ```
 
   * 执行命令，在集群中部署cr实例后，节点会根据配置的参数信息进行配置，再次查看节点状态(NODESTATUS变成config)
 
     ```shell
     kubectl apply -f upgrade_v1alpha1_os.yaml
-    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version'
+    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version'
     ```
 
   * 再次查看节点的配置的版本确认节点是否配置完成(NODESTATUS恢复为idle)
 
     ```shell
-    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradeconfigs.version'
+    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version'
     ```
 
-* 如果后续需要再次升级，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。
+* 如果后续需要再次配置，与上面相同对 upgrade_v1alpha1_os.yaml 的相应字段进行相应修改。
 
 #### 回退指导
 
 * 回退场景
-  * 虚拟机无法正常启动时，需要退回到上一可以启动的版本时进行回退操作，仅支持手动回退容器 OS 。
-  * 虚拟机能够正常启动并且进入系统，需要将当前版本退回到老版本时进行回退操作，支持工具回退（类似升级方式）和手动回退，建议使用工具回退。
-  * 配置出现错误，节点状态陷入config时，可以回退至上一个配置版本以恢复节点至idle状态。
-  **注意**：在配置新版本时，出现错误前已经配置的参数无法回退。
+  * 虚拟机无法正常启动时，可在grub启动项页面手动切换启动项，使系统回退至上一版本（即手动回退）。
+  * 虚拟机能够正常启动并且进入系统时，支持工具回退和手动回退，建议使用工具回退。
+  * 工具回退有两种方式：
+    1. rollback模式直接回退至上一版本。
+    2. upgrade模式重新升级至上一版本
 * 手动回退指导
-  * 手动重启虚拟机，选择第二启动项进行回退，手动回退仅支持回退到本次升级之前的版本。
+  
+  * 手动重启虚拟机，进入启动项页面后，选择第二启动项进行回退，手动回退仅支持回退到上一个版本。
 * 工具回退指导
   * 回退至任意版本
-  * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。
+    * 修改 OS 的cr实例的YAML 配置文件（例如 upgrade_v1alpha1_os.yaml），设置相应字段为期望回退的老版本镜像信息。类别OS来自于安装和部署章节创建的CRD对象，字段说明及示例请见上一节升级指导。
+
     * YAML修改完成后执行更新命令，在集群中更新定制对象后，节点会根据配置的字段信息进行回退
 
         ```shell
@@ -499,36 +594,29 @@
   * 查看节点容器 OS 版本(回退OS版本)或节点config版本&节点状态为idle(回退config版本)，确认回退是否成功。
 
     ```shell
-    kubectl get nodes -o custom-columns='NAME:.metadata.name,OS:.status.nodeInfo.osImage'
-
-    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADESYSCONFIG:status.upgradesysconfigs.version'
+    kubectl get osinstances -o custom-columns='NAME:.metadata.name,NODESTATUS:.spec.nodestatus,SYSCONFIG:status.sysconfigs.version,UPGRADECONFIG:status.upgradeconfigs.version'
     ```
 
-#### Admin容器
+## Admin容器镜像制作、部署和使用
 
 KubeOS提供一个分离的包含sshd服务和hostshell工具的Admin容器，来帮助管理员在必要情况下登录KubeOS，其中的sshd服务由[sysmaster](https://gitee.com/openeuler/sysmaster)/systemd拉起。Admin容器部署后用户可通过ssh连接到节点的Admin容器，进入Admin容器后执行hostshell命令获取host的root shell。
 
-##### 部署方法
+### admin容器镜像制作
 
-以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/everything/aarch64/Packages/)到scripts/admin-container目录下。
+以sysmaster为例，根据系统版本和架构，获取对应的sysmaster RPM包，如获取openEuler-22.03-LTS-SP1-aarch64版本的[sysmaster](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/update/aarch64/Packages/)到scripts/admin-container目录下。
 
-**修改**admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP2/docker_img)下载。
+修改admin-container目录下的Dockerfile，指定sysmaster RPM包的路径，其中的openeuler-22.03-lts-sp1可在[openEuler Repo](https://repo.openeuler.org/openEuler-22.03-LTS-SP1/docker_img)下载。
 
 ```Dockerfile
-FROM openeuler-22.03-lts
-
+FROM openeuler-22.03-lts-sp1
 RUN yum -y install openssh-clients util-linux
-
 ADD ./your-sysmaster.rpm /home
 RUN rpm -ivh  /home/your-sysmaster.rpm
-
 COPY ./hostshell /usr/bin/
 COPY ./set-ssh-pub-key.sh /usr/local/bin
 COPY ./set-ssh-pub-key.service /usr/lib/sysmaster
-
 EXPOSE 22
 RUN sed -i 's/sysinit.target/sysinit.target;sshd.service;set-ssh-pub-key.service/g' /usr/lib/sysmaster/basic.target
-
 CMD ["/usr/lib/sysmaster/init"]
 ```
 
@@ -546,7 +634,9 @@ bash -x kbimg.sh create admin-image -f admin-container/Dockerfile -d your_imageR
 docker push your_imageRepository/admin_imageName:version
 ```
 
-在master节点上部署Admin容器，需要提供ssh公钥来免密登录，**修改**并应用如下示例yaml文件:
+### admin容器部署
+
+在master节点上部署Admin容器，需要提供ssh公钥来免密登录，修改并应用如下示例yaml文件:
 
 ```yaml
 apiVersion: v1
@@ -554,8 +644,7 @@ kind: Secret
 metadata:
   name: root-secret
 data:
-  # base64 encode your pub key in one line
-  ssh-pub-key: your-ssh-pub-key
+  ssh-pub-key: <your-ssh-pub-key-encoded-with-base64>
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -563,7 +652,7 @@ metadata:
   name: admin-container-sysmaster
   namespace: default
   labels:
-      control-plane: admin-container-sysmaster
+    control-plane: admin-container-sysmaster
 spec:
   selector:
     matchLabels:
@@ -576,24 +665,24 @@ spec:
     spec:
       hostPID: true
       containers:
-      - name: admin-container-sysmaster
-        image: your_imageRepository/admin_imageName:version
-        imagePullPolicy: Always
-        securityContext:
-          privileged: true
-        ports:
-          - containerPort: 22
-        # sysmaster要求
-        env:
-          - name: container
-            value: containerd
-        volumeMounts:
-        # name 必须与下面的卷名匹配
-        - name: secret-volume
-        # mountPath必须为/etc/secret-volume
-          mountPath: /etc/secret-volume
-          readOnly: true
-      nodeName: your-worker-node-name
+        - name: admin-container-sysmaster
+          image: <your_imageRepository/admin_imageName:version>
+          imagePullPolicy: Always
+          securityContext:
+            privileged: true
+          ports:
+            - containerPort: 22
+          # sysmaster要求
+          env:
+            - name: container
+              value: containerd
+          volumeMounts:
+            # name 必须与下面的卷名匹配
+            - name: secret-volume
+              # mountPath必须为/etc/secret-volume
+              mountPath: /etc/secret-volume
+              readOnly: true
+      nodeName: <your-worker-node-name>
       volumes:
         - name: secret-volume
           secret:
@@ -610,11 +699,13 @@ spec:
   ports:
     - port: 22
       targetPort: 22
-      nodePort: your-exposed-port
+      nodePort: <your-exposed-port>
   selector:
-      control-plane: admin-container-sysmaster
+    control-plane: admin-container-sysmaster
 ```
 
+### admin容器使用
+
 ssh到Admin容器，然后执行hostshell命令进入host root shell, 如：
 
 ```shell
@@ -622,7 +713,7 @@ ssh -p your-exposed-port root@your.worker.node.ip
 hostshell
 ```
 
-##### hostshell
+#### hostshell说明
 
 为了保证KubeOS的轻便性，许多工具或命令没有安装在KubeOS内。因此，用户可以在制作Admin容器时，将期望使用的二进制文件放在容器内的如/usr/bin目录下。hostshell工具在执行时会将容器下的/usr/bin, /usr/sbin, /usr/local/bin, /usr/local/sbin路径添加到host root shell的环境变量。
 
@@ -637,7 +728,7 @@ hostshell
 
 #### kernel Settings
 
-* kenerl.sysctl: 临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）, 示例如下:
+* kenerl.sysctl：临时设置内核参数，重启后无效，key/value 表示内核参数的 key/value， key与value均不能为空且key不能包含“=”，该参数不支持删除操作（operation=delete）示例如下:
 
     ```yaml
     configs:
@@ -668,23 +759,38 @@ hostshell
 * grub.cmdline: 设置grub.cfg文件中的内核引导参数，该行参数在grub.cfg文件中类似如下示例：
 
   ```shell
-      linux   /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3
-  ```  
+  linux   /boot/vmlinuz root=/dev/sda2 ro rootfstype=ext4 nomodeset quiet oops=panic softlockup_panic=1 nmi_watchdog=1 rd.shell=0 selinux=0 crashkernel=256M panic=3
+  ```
 
 * KubeOS使用双分区，grub.cmdline支持对当前分区或下一分区进行配置：
+
   * grub.cmdline.current：对当前分区的启动项参数进行配置。
   * grub.cmdline.next：对下一分区的启动项参数进行配置。
+
 * 注意：升级/回退前后的配置，始终基于升级/回退操作下发时的分区位置进行current/next的区分。假设当前分区为A分区，下发升级操作并在sysconfigs（升级重启后配置）中配置grub.cmdline.current，重启后进行配置时仍修改A分区对应的grub cmdline。
+
 * grub.cmdline.current/next支持“key=value”（value不能为空），也支持单key。若value中有“=”，例如“root=UUID=some-uuid”，key应设置为第一个“=”前的所有字符，value为第一个“=”后的所有字符。 配置方法示例如下：
+
     ```yaml
     configs:
-      - model: grub.cmdline.current
-        contents:
-            - key: selinux
-              value: 0
-            - key: root
-              value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94
-            - key: panic
-              value: 3
-              operation: delete
+    - model: grub.cmdline.current
+      contents:
+          - key: selinux
+            value: "0"
+          - key: root
+            value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94
+          - key: panic
+            value: "3"
+            operation: delete
+          - key: crash_kexec_post_notifiers
+    - model: grub.cmdline.next
+      contents:
+          - key: selinux
+            value: "0"
+          - key: root
+            value: UUID=e4f1b0a0-590e-4c5f-9d8a-3a2c7b8e2d94
+          - key: panic
+            value: "3"
+            operation: delete
+          - key: crash_kexec_post_notifiers
     ```
diff --git "a/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md" "b/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md"
index f2823c85542c90ae6deb79e7885487768673ee5d..d54cc837d5704e2746a6ecccea267b11e8f0cdc5 100644
--- "a/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md"
+++ "b/docs/user_guide/\345\256\271\345\231\250OS\351\225\234\345\203\217\345\210\266\344\275\234\346\214\207\345\257\274.md"
@@ -1,16 +1,16 @@
-# 容器OS镜像制作指导#
+# 容器OS镜像制作指导
 
-## 简介 ##
+## 简介
 
 kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制作KubeOS 容器，虚拟机和物理机镜像
 
-## 命令介绍 ##
+## 命令介绍
 
-### 命令格式 ###
+### 命令格式
 
 **bash kbimg.sh** \[ --help | -h \] create \[ COMMANDS \]  \[ OPTIONS \]
 
-### 参数说明 ###
+### 参数说明
 
 * COMMANDS
 
@@ -20,8 +20,6 @@ kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制
   | vm-image      | 生成用于部署和升级的虚拟机镜像              |
   | pxe-image     | 生成物理机安装所需的镜像及文件              |
 
-
-
 * OPTIONS
 
   | 参数         | 描述                                                         |
@@ -34,31 +32,33 @@ kbimg是KubeOS部署和升级所需的镜像制作工具，可以使用kbimg制
   | -l           | 如果指定参数，则镜像为legacy引导，不指定默认是UEFI引导               |
   | -h  --help | 查看帮助信息                                                 |
 
+## 使用说明
 
-
-## 使用说明 ##
-
-#### 注意事项 ###
+### 注意事项
 
 * kbimg.sh 执行需要 root 权限
 * 当前仅支持 x86和 AArch64 架构使用
 * 容器 OS 镜像制作工具的 rpm 包源为 openEuler 具体版本的 everything 仓库和 EPOL 仓库。制作镜像时提供的 repo 文件中，yum 源建议同时配置 openEuler 具体版本的 everything 仓库和 EPOL 仓库
 
-### KubeOS OCI 镜像制作 ###
+### KubeOS OCI 镜像制作
 
-#### 注意事项 ####
+#### 注意事项
 
 * 制作的 OCI 镜像仅用于后续的虚拟机/物理机镜像制作或升级使用，不支持启动容器
 * 使用默认 rpmlist 进行容器OS镜像制作时所需磁盘空间至少为6G，如自已定义 rpmlist 可能会超过6G
 
-#### 使用示例 ####
+#### 使用示例
+
 * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件
+
 ```shell
   cd /opt/kubeOS/scripts
   touch resolv.conf
   vim resolv.conf
 ```
+
 * 制作KubeOS容器镜像
+
 ``` shell
 cd /opt/kubeOS/scripts
 bash kbimg.sh create upgrade-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0''' -d your_imageRepository/imageName:version 
@@ -70,25 +70,28 @@ bash kbimg.sh create upgrade-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1
 docker images
 ```
 
-### KubeOS 虚拟机镜像制作 ###
+### KubeOS 虚拟机镜像制作
 
-#### 注意事项 ####
+#### 注意事项
 
 * 如使用 docker 镜像制作请先拉取相应镜像或者先制作docker镜像，并保证 docker 镜像的安全性
 * 制作出来的容器 OS 虚拟机镜像目前只能用于 CPU 架构为 x86 和 AArch64 的虚拟机
 * 容器 OS 目前不支持 x86 架构的虚拟机使用 legacy 启动模式启动
 * 使用默认rpmlist进行容器OS镜像制作时所需磁盘空间至少为25G，如自已定义rpmlist可能会超过25G
 
-#### 使用示例 ####
+#### 使用示例
 
 * 使用repo源制作
-    * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件
+  * 如需进行DNS配置，请先在```scripts```目录下自定义```resolv.conf```文件
+
   ```shell
   cd /opt/kubeOS/scripts
   touch resolv.conf
   vim resolv.conf
   ```
-    * KubeOS虚拟机镜像制作
+
+  * KubeOS虚拟机镜像制作
+
   ``` shell
   cd /opt/kubeOS/scripts
   bash kbimg.sh create vm-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0'''
@@ -100,15 +103,15 @@ docker images
   cd /opt/kubeOS/scripts
   bash kbimg.sh create vm-image -d  your_imageRepository/imageName:version
   ```
-* 结果说明    
-  容器 OS 镜像制作完成后，会在 /opt/kubeOS/scripts 目录下生成：
-    * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。
-    * update.img: 用于升级的根文件系统分区镜像
 
+* 结果说明
+  容器 OS 镜像制作完成后，会在 /opt/kubeOS/scripts 目录下生成：
+  * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。
+  * update.img: 用于升级的根文件系统分区镜像
 
-### KubeOS 物理机安装所需镜像及文件制作 ###
+### KubeOS 物理机安装所需镜像及文件制作
 
-#### 注意事项 ####
+#### 注意事项
 
 * 如使用 docker 镜像制作请先拉取相应镜像或者先制作 docker 镜像，并保证 docker 镜像的安全性
 * 制作出来的容器 OS 物理安装所需的镜像目前只能用于 CPU 架构为 x86 和 AArch64 的物理机安装
@@ -116,7 +119,8 @@ docker images
 * 不支持多个磁盘都安装KubeOS，可能会造成启动失败或挂载紊乱
 * 容器OS 目前不支持 x86 架构的物理机使用 legacy 启动模式启动
 * 使用默认rpmlist进行镜像制作时所需磁盘空间至少为5G，如自已定义 rpmlist 可能会超过5G
-#### 使用示例 ####
+
+#### 使用示例
 
 * 首先需要修改```00bootup/Global.cfg```的配置，对相关参数进行配置，参数均为必填，ip目前仅支持ipv4，配置示例如下
 
@@ -138,25 +142,344 @@ docker images
   ```
 
 * 使用 repo 源制作
-    * 如需进行DNS配置，请在```scripts```目录下自定义```resolv.conf```文件
+  * 如需进行DNS配置，请在```scripts```目录下自定义```resolv.conf```文件
+
   ```shell
     cd /opt/kubeOS/scripts
     touch resolv.conf
     vim resolv.conf
   ```
-    * KubeOS物理机安装所需镜像制作
-  ```
+
+  * KubeOS物理机安装所需镜像制作
+
+  ```shell
     cd /opt/kubeOS/scripts
     bash kbimg.sh create pxe-image -p xxx.repo -v v1 -b ../bin/os-agent -e '''$1$xyz$RdLyKTL32WEvK3lg8CXID0'''
   ```
 
 * 使用 docker 镜像制作
+
   ``` shell
   cd /opt/kubeOS/scripts
   bash kbimg.sh create pxe-image -d your_imageRepository/imageName:version
   ```
 
 * 结果说明
+  * initramfs.img: 用于pxe启动用的 initramfs 镜像
+  * kubeos.tar: pxe安装所用的 OS
+
+
+# KubeOS-Rust 镜像制作说明
+
+## 简介
+
+KubeOS 虚拟机镜像制作的 Rust 二进制版本
+
+## 命令介绍
+
+### 命令格式
+
+**.../kbimg** \[ --config | -c \] \<path to kbimg.toml\>
+
+## 配置文件说明
+
+* from_repo: 从 repo 创建 OCI 镜像、虚拟机镜像或物理机镜像
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | agent_path | os-agent 二进制的路径 |
+  | image_type | upgrade: 用于安装和升级的 OCI 镜像格式的 KubeOS 镜像; vm-repo: 用于部署和升级的虚拟机镜像; pxe-repo: 物理机安装所需的镜像及文件 |
+  | legacy_bios | 镜像为 legacy 引导或 UEFI 引导 |
+  | repo_path | repo 文件的路径，repo 文件中配置制作镜像所需要的 yum 源 |
+  | root_passwd | KubeOS 镜像 root 用户密码，加密后的带盐值的密码，可以用 openssl、kiwi 命令生成 |
+  | version | 制作出来的 KubeOS 镜像的版本 |
+  | rpmlist | 镜像所需的 rpm 包 |
+  | docker_img | 生成或者使用的 docker 镜像 |
+
+* from_docker: 从 docker 镜像创建虚拟机镜像或物理机镜像
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | docker_img | 生成或者使用的 docker 镜像 |
+  | image_type | vm-docker: 用于部署和升级的虚拟机镜像; pxe-docker: 物理机安装所需的镜像及文件 |
+
+* admin_container: 
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | dockerfile | dockerfile 路径 |
+  | docker_img | 生成或者使用的 docker 镜像 |
+
+* [OPTIONAL] users: 添加用户
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | groups | [OPTIONAL] 用户组 (第一个为主组，其他为附加组) |
+  | name | 用户名 |
+  | passwd | 密码 |
+  | sudo | [OPTIONAL] 用户是否具有 sudo 权限 |
+
+* [OPTIONAL] copy_files: 拷贝文件到指定目录
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | dst | 目标目录 |
+  | src | 源文件路径 |
+
+* [OPTIONAL] grub: grub配置
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | passwd | [OPTIONAL] grub 密码 |
+
+* [OPTIONAL] systemd_service: 新增 systemd 服务
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | name | systemd 服务名 |
+
+* [OPTIONAL] chroot_script: 自定义 chroot 脚本
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | path | 脚本路径 |
+
+* [OPTIONAL] disk_partition: 自定义分区大小和镜像大小
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | first | 引导分区大小 |
+  | second | ROOT-A 分区大小 | 
+  | third | ROOT-B 分区大小 |
+  | img_size | 镜像大小 |
+
+* [OPTIONAL] persist_mkdir: persist 分区新建目录
+
+  | 参数 | 描述 |
+  | --- | --- |
+  | name | 目录名 |
+
+## 使用说明
+
+#### 注意事项
+
+* 新增 systemd 服务需要将对应的 .service 文件或 .mount 文件拷贝至镜像```/usr/lib/systemd/system```目录
+
+  ```toml
+  [[copy_files]]
+  dst = "/usr/lib/systemd/system"
+  src = ".../containerd.service"
+
+  [systemd_service]
+  name = ["containerd"]
+  ```
+
+  * 如需挂载数据盘，请先自定义```persist-data.mount```文件，并启用```copy_files```和```systemd_service```字段设置启动时挂载，启用```persist_mkdir```字段创建挂载点
+  * .mount文件名由挂载点路径生成，将斜杠替换为连字符
+  * 请先在磁盘映像文件上创建ext4文件系统
 
-    * initramfs.img: 用于pxe启动用的 initramfs 镜像
-    * kubeos.tar: pxe安装所用的 OS
+    ```
+    # persist-data.mount
+    [Unit]
+    Description=Mount Disk
+    Documentation=man:systemd.mount(5)
+
+    [Mount]
+    What=/dev/vdb
+    Where=/persist/data
+    Type=ext4
+    Options=defaults,noatime
+
+    [Install]
+    WantedBy=local-fs.target
+    ```
+
+    ```toml
+    [[copy_files]]
+    dst = "/usr/lib/systemd/system"
+    src = ".../persist-data.mount"
+
+    [systemd_service]
+    name = ["persist-data.mount"]
+
+    [persist_mkdir]
+    name = ["data"]
+    ```
+
+  * 如需配置逻辑卷，请先自定义```volume.service```文件，并启用```copy_files```和```systemd_service```设置启动时配置逻辑卷，启用```persist_mkdir```字段创建挂载点
+
+    ```
+    # volume.service
+    [Unit]
+    Description=Mount Logical Volume
+    After=local-fs.target
+
+    [Service]
+    Type=oneshot
+    RemainAfterExit=yes
+    ExecStart=pvcreate /dev/vdb
+    ExecStart=pvcreate /dev/vdc
+    ExecStart=vgcreate my_vg /dev/vdb /dev/vdc
+    ExecStart=lvcreate -L 15G -n my_lv my_vg
+    ExecStart=mkfs.ext4 /dev/my_vg/my_lv
+    ExecStart=mount /dev/my_vg/my_lv /persist/lv_data
+
+    [Install]
+    WantedBy=local-fs.target
+    ```
+
+    ```toml
+    [[copy_files]]
+    dst = "/usr/lib/systemd/system"
+    src = ".../volume.service"
+
+    [systemd_service]
+    name = ["volume"]
+
+    [persist_mkdir]
+    name = ["lv_data"]
+    ```
+
+## 使用示例
+
+### KubeOS OCI 镜像制作
+
+* 如需进行DNS配置，请先自定义```resolv.conf```文件，并启用```copy_files```字段将配置文件拷贝到```/etc```目录
+
+  ```shell
+  touch \<arbitrary directory\>/resolv.conf
+  vim \<arbitrary directory\>resolv.conf
+  ```
+
+  ```toml
+  [[copy_files]]
+  dst = "/etc"
+  src = "<path to resolv.conf>"
+  ```
+
+* 制作KubeOS容器镜像
+
+  ```toml
+  [from_repo]
+  agent_path = "<directory>/bin/os-agent"
+  image_type = "upgrade"
+  legacy_bios = false
+  repo_path = "xxx.repo"
+  root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0"
+  version = "v1"
+  docker_img = "your_imageRepository/imageName:version"
+  rpmlist = [
+      # your rpms
+  ]
+  ```
+
+* 制作完成后查看制作出来的KubeOS容器镜像
+  ``` shell
+  docker images
+  ```
+
+### KubeOS 虚拟机镜像制作
+
+* 使用repo源制作
+
+    * 如需进行DNS配置，请先自定义```resolv.conf```文件，并启用**copy_files**字段将配置文件拷贝到```/etc```目录
+    
+      ```shell
+      touch \<arbitrary directory\>/resolv.conf
+      vim \<arbitrary directory\>resolv.conf
+      ```
+
+      ```toml
+      [[copy_files]]
+      dst = "/etc"
+      src = "<path to resolv.conf>"
+      ```
+
+    * KubeOS虚拟机镜像制作
+
+      ```toml
+      [from_repo]
+      agent_path = "<directory>/bin/os-agent"
+      image_type = "vm-repo"
+      legacy_bios = false
+      repo_path = "xxx.repo"
+      root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0"
+      version = "v1"
+      rpmlist = [
+          # your rpms
+      ]
+      ```
+
+* 使用docker镜像制作
+
+  ```toml
+  [from_dockerimg]
+  docker_img = "your_imageRepository/imageName:version"
+  image_type = "vm-docker"
+  ```
+
+* 结果说明    
+  容器 OS 镜像制作完成后，会在 ./scripts-auto 目录下生成
+    * system.qcow2: qcow2 格式的系统镜像，大小默认为 20GiB，支持的根文件系统分区大小 < 2020 MiB，持久化分区 < 16GiB 。
+    * update.img: 用于升级的根文件系统分区镜像
+
+### KubeOS 物理机安装所需镜像及文件制作
+
+* 首先需要修改```kbimg.toml```中```pxe_config```的配置，对相关参数进行配置，参数均为必填，ip目前仅支持ipv4，配置示例如下
+
+  ```toml
+  [pxe_config]
+  # rootfs file name
+  rootfs_name = "kubeos.tar"
+  # select the target disk to install kubeOS
+  disk = "/dev/sda"
+  # pxe server ip address where stores the rootfs on the http server
+  server_ip = "192.168.1.50"
+  # target machine ip
+  local_ip = "192.168.1.100"
+  # target machine route
+  route_ip = "192.168.1.1"
+  # target machine netmask
+  netmask = "255.255.255.0"
+  # target machine netDevice name
+  net_name = "eth0"
+  ```
+
+* 使用 repo 源制作
+  * 如需进行DNS配置，请先自定义```resolv.conf```文件，并启用```copy_files```字段将配置文件拷贝到```/etc```目录
+    
+    ```shell
+    touch \<arbitrary directory\>/resolv.conf
+    vim \<arbitrary directory\>resolv.conf
+    ```
+
+    ```toml
+    [[copy_files]]
+    dst = "/etc"
+    src = "<path to resolv.conf>"
+    ```
+
+  * KubeOS物理机安装所需镜像制作
+    ```toml
+    [from_repo]
+    agent_path = "<directory>/bin/os-agent"
+    image_type = "pxe-repo"
+    legacy_bios = true
+    repo_path = "xxx.repo"
+    root_passwd = "$1$xyz$RdLyKTL32WEvK3lg8CXID0"
+    version = "v1"
+    rpmlist = [
+        # your rpms
+    ]
+    ```
+
+* 使用 docker 镜像制作
+  ```toml
+  [from_dockerimg]
+  docker_img = "your_imageRepository/imageName:version"
+  image_type = "vm-docker"
+  ```
+
+* 结果说明
+  * initramfs.img: 用于pxe启动用的 initramfs 镜像
+  * kubeos.tar: pxe安装所用的 OS
diff --git a/pkg/values/values.go b/pkg/values/values.go
index e2c035407e80f05bde84a4e6e59fad277cb671b4..9e213a8344216206ac62aa02feb87e93d2a3a8ca 100644
--- a/pkg/values/values.go
+++ b/pkg/values/values.go
@@ -30,13 +30,18 @@ const (
 	LabelNodeSelector = "upgrade.openeuler.org/node-selector"
 	// LabelConfiguring is the key of the configuring label for nodes
 	LabelConfiguring = "upgrade.openeuler.org/configuring"
-	defaultPeriod    = 15 * time.Second
+	// LabelSerial is the key of the serial label for nodes
+	LabelSerial = "upgrade.openeuler.org/serial"
+
+	defaultPeriod = 15 * time.Second
 	// OsiStatusName is param name of nodeStatus in osInstance
 	OsiStatusName = "nodestatus"
 	// UpgradeConfigName is param name of UpgradeConfig
 	UpgradeConfigName = "UpgradeConfig"
 	// SysConfigName is param name of SysConfig
 	SysConfigName = "SysConfig"
+	// OsiNamespace is the namespace of osinstance
+	OsiNamespace = "default"
 )
 
 // NodeStatus defines state of nodes
diff --git a/scripts/bootloader.sh b/scripts/bootloader.sh
index 75096a38b01b214e07b111b40d6f465dad40d1da..df4be3290121f5ca6b3426e1c291c3d446f71aeb 100644
--- a/scripts/bootloader.sh
+++ b/scripts/bootloader.sh
@@ -19,7 +19,7 @@ function install_grub2_x86 ()
         cp -r /usr/lib/grub/x86_64-efi boot/efi/EFI/openEuler
         eval "grub2-mkimage -d /usr/lib/grub/x86_64-efi -O x86_64-efi --output=/boot/efi/EFI/openEuler/grubx64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux"
 
-        mkdir -p /boot/EFI/BOOT/
+        mkdir -p /boot/efi/EFI/BOOT/
         cp -f /boot/efi/EFI/openEuler/grubx64.efi /boot/efi/EFI/BOOT/BOOTX64.EFI
     fi
 }
@@ -29,7 +29,7 @@ function install_grub2_efi ()
     cp -r /usr/lib/grub/arm64-efi /boot/efi/EFI/openEuler/
     eval "grub2-mkimage -d /usr/lib/grub/arm64-efi -O arm64-efi --output=/boot/efi/EFI/openEuler/grubaa64.efi '--prefix=(,gpt1)/EFI/openEuler' fat part_gpt part_msdos linux"
 
-    mkdir -p /boot/EFI/BOOT/
+    mkdir -p /boot/efi/EFI/BOOT/
     cp -f /boot/efi/EFI/openEuler/grubaa64.efi /boot/efi/EFI/BOOT/BOOTAA64.EFI
 }